From 40a25cdc13c3439c2f23a1cfc2c86426ba5b3369 Mon Sep 17 00:00:00 2001
From: Andrew Fitzgerald <apfitzge@gmail.com>
Date: Tue, 2 Aug 2022 10:22:59 -0500
Subject: [PATCH 0001/3199] benchmarking code for transaction scheduler

---
 Cargo.lock                              |  70 ++--
 Cargo.toml                              |   1 +
 transaction-scheduler-bench/Cargo.toml  |  26 ++
 transaction-scheduler-bench/src/main.rs | 499 ++++++++++++++++++++++++
 4 files changed, 573 insertions(+), 23 deletions(-)
 create mode 100644 transaction-scheduler-bench/Cargo.toml
 create mode 100644 transaction-scheduler-bench/src/main.rs

diff --git a/Cargo.lock b/Cargo.lock
index 14808c6088545d..8ef029497215fa 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -801,16 +801,16 @@ dependencies = [
 
 [[package]]
 name = "clap"
-version = "3.1.8"
+version = "3.2.16"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "71c47df61d9e16dc010b55dba1952a57d8c215dbb533fd13cdd13369aac73b1c"
+checksum = "a3dbbb6653e7c55cc8595ad3e1f7be8f32aba4eb7ff7f0fd1163d4f3d137c0a9"
 dependencies = [
  "atty",
  "bitflags",
  "clap_derive",
+ "clap_lex",
  "indexmap",
- "lazy_static",
- "os_str_bytes",
+ "once_cell",
  "strsim 0.10.0",
  "termcolor",
  "textwrap 0.15.0",
@@ -818,9 +818,9 @@ dependencies = [
 
 [[package]]
 name = "clap_derive"
-version = "3.1.7"
+version = "3.2.15"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a3aab4734e083b809aaf5794e14e756d1c798d2c69c7f7de7a09a2f5214993c1"
+checksum = "9ba52acd3b0a5c33aeada5cdaa3267cdc7c594a98731d4268cdc1532f4264cb4"
 dependencies = [
  "heck 0.4.0",
  "proc-macro-error",
@@ -829,6 +829,15 @@ dependencies = [
  "syn 1.0.98",
 ]
 
+[[package]]
+name = "clap_lex"
+version = "0.2.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2850f2f5a82cbf437dd5af4d49848fbdfc27c157c3d010345776f952765261c5"
+dependencies = [
+ "os_str_bytes",
+]
+
 [[package]]
 name = "cmake"
 version = "0.1.46"
@@ -3011,9 +3020,6 @@ name = "os_str_bytes"
 version = "6.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "8e22443d1643a904602595ba1cd8f7d896afe56d26712531c5ff73a15b2fbf64"
-dependencies = [
- "memchr",
-]
 
 [[package]]
 name = "ouroboros"
@@ -3785,7 +3791,7 @@ dependencies = [
 name = "rbpf-cli"
 version = "1.11.5"
 dependencies = [
- "clap 3.1.8",
+ "clap 3.2.16",
  "serde",
  "serde_json",
  "solana-bpf-loader-program",
@@ -4597,7 +4603,7 @@ dependencies = [
 name = "solana-banking-bench"
 version = "1.11.5"
 dependencies = [
- "clap 3.1.8",
+ "clap 3.2.16",
  "crossbeam-channel",
  "log",
  "rand 0.7.3",
@@ -4664,7 +4670,7 @@ dependencies = [
 name = "solana-bench-streamer"
 version = "1.11.5"
 dependencies = [
- "clap 3.1.8",
+ "clap 3.2.16",
  "crossbeam-channel",
  "solana-net-utils",
  "solana-streamer",
@@ -4771,7 +4777,7 @@ name = "solana-cargo-build-bpf"
 version = "1.11.5"
 dependencies = [
  "cargo_metadata",
- "clap 3.1.8",
+ "clap 3.2.16",
  "solana-sdk 1.11.5",
 ]
 
@@ -4781,7 +4787,7 @@ version = "1.11.5"
 dependencies = [
  "bzip2",
  "cargo_metadata",
- "clap 3.1.8",
+ "clap 3.2.16",
  "log",
  "regex",
  "serial_test",
@@ -4796,7 +4802,7 @@ name = "solana-cargo-test-bpf"
 version = "1.11.5"
 dependencies = [
  "cargo_metadata",
- "clap 3.1.8",
+ "clap 3.2.16",
 ]
 
 [[package]]
@@ -4804,7 +4810,7 @@ name = "solana-cargo-test-sbf"
 version = "1.11.5"
 dependencies = [
  "cargo_metadata",
- "clap 3.1.8",
+ "clap 3.2.16",
 ]
 
 [[package]]
@@ -4829,7 +4835,7 @@ name = "solana-clap-v3-utils"
 version = "1.11.5"
 dependencies = [
  "chrono",
- "clap 3.1.8",
+ "clap 3.2.16",
  "rpassword",
  "solana-perf",
  "solana-remote-wallet",
@@ -5103,7 +5109,7 @@ name = "solana-dos"
 version = "1.11.5"
 dependencies = [
  "bincode",
- "clap 3.1.8",
+ "clap 3.2.16",
  "crossbeam-channel",
  "itertools",
  "log",
@@ -5416,7 +5422,7 @@ name = "solana-keygen"
 version = "1.11.5"
 dependencies = [
  "bs58",
- "clap 3.1.8",
+ "clap 3.2.16",
  "dirs-next",
  "num_cpus",
  "solana-clap-v3-utils",
@@ -5557,7 +5563,7 @@ name = "solana-log-analyzer"
 version = "1.11.5"
 dependencies = [
  "byte-unit",
- "clap 3.1.8",
+ "clap 3.2.16",
  "serde",
  "serde_json",
  "solana-logger 1.11.5",
@@ -5634,7 +5640,7 @@ dependencies = [
 name = "solana-net-shaper"
 version = "1.11.5"
 dependencies = [
- "clap 3.1.8",
+ "clap 3.2.16",
  "rand 0.7.3",
  "serde",
  "serde_json",
@@ -5646,7 +5652,7 @@ name = "solana-net-utils"
 version = "1.11.5"
 dependencies = [
  "bincode",
- "clap 3.1.8",
+ "clap 3.2.16",
  "crossbeam-channel",
  "log",
  "nix",
@@ -5723,7 +5729,7 @@ dependencies = [
 name = "solana-poh-bench"
 version = "1.11.5"
 dependencies = [
- "clap 3.1.8",
+ "clap 3.2.16",
  "log",
  "rand 0.7.3",
  "rayon",
@@ -6406,6 +6412,24 @@ dependencies = [
  "solana-version",
 ]
 
+[[package]]
+name = "solana-transaction-scheduler-bench"
+version = "1.11.5"
+dependencies = [
+ "bincode",
+ "clap 3.2.16",
+ "crossbeam-channel",
+ "itertools",
+ "log",
+ "rand 0.8.5",
+ "solana-core",
+ "solana-logger 1.11.5",
+ "solana-measure",
+ "solana-perf",
+ "solana-runtime",
+ "solana-sdk 1.11.5",
+]
+
 [[package]]
 name = "solana-transaction-status"
 version = "1.11.5"
diff --git a/Cargo.toml b/Cargo.toml
index ef847c1024a17d..17b2af99a52b80 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -81,6 +81,7 @@ members = [
     "tokens",
     "transaction-dos",
     "transaction-status",
+    "transaction-scheduler-bench",
     "upload-perf",
     "validator",
     "version",
diff --git a/transaction-scheduler-bench/Cargo.toml b/transaction-scheduler-bench/Cargo.toml
new file mode 100644
index 00000000000000..d963b62ac5bfaf
--- /dev/null
+++ b/transaction-scheduler-bench/Cargo.toml
@@ -0,0 +1,26 @@
+[package]
+authors = ["Solana Maintainers <maintainers@solana.foundation>"]
+edition = "2021"
+name = "solana-transaction-scheduler-bench"
+version = "1.11.5"
+repository = "https://github.com/solana-labs/solana"
+license = "Apache-2.0"
+homepage = "https://solana.com/"
+documentation = "https://docs.rs/solana-transaction-scheduler-bench"
+publish = false
+
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+
+[dependencies]
+clap = { version = "3.1.18", features = ["derive", "env"] }
+bincode = "1.3.3"
+crossbeam-channel = "0.5"
+itertools = "0.10.3"
+log = "0.4.17"
+rand = "0.8.5"
+solana-core = { path = "../core" }
+solana-logger = { path = "../logger" }
+solana-measure = { path = "../measure" }
+solana-perf = { path = "../perf" }
+solana-runtime = { path = "../runtime" }
+solana-sdk = { path = "../sdk" }
\ No newline at end of file
diff --git a/transaction-scheduler-bench/src/main.rs b/transaction-scheduler-bench/src/main.rs
new file mode 100644
index 00000000000000..a795dc5ed5acae
--- /dev/null
+++ b/transaction-scheduler-bench/src/main.rs
@@ -0,0 +1,499 @@
+use {
+    clap::Parser,
+    crossbeam_channel::{select, Receiver, Sender},
+    log::info,
+    rand::Rng,
+    solana_core::transaction_priority_details::GetTransactionPriorityDetails,
+    solana_measure::measure,
+    solana_perf::packet::{Packet, PacketBatch},
+    solana_runtime::{bank::Bank, bank_forks::BankForks},
+    solana_sdk::{
+        compute_budget::ComputeBudgetInstruction,
+        hash::Hash,
+        instruction::{AccountMeta, Instruction},
+        signature::Keypair,
+        signer::Signer,
+        system_program,
+        transaction::{SanitizedTransaction, Transaction, VersionedTransaction},
+    },
+    std::{
+        sync::{
+            atomic::{AtomicBool, AtomicU64, AtomicUsize, Ordering},
+            Arc, RwLock,
+        },
+        thread::{sleep, JoinHandle},
+        time::{Duration, Instant},
+    },
+};
+
+#[derive(Parser, Debug)]
+#[clap(author, version, about, long_about = None)]
+struct Args {
+    /// How many packets per second to send to the scheduler
+    #[clap(long, env, default_value_t = 200_000)]
+    packet_send_rate: usize,
+
+    /// Number of packets per batch
+    #[clap(long, env, default_value_t = 128)]
+    packets_per_batch: usize,
+
+    /// Number of batches per message
+    #[clap(long, env, default_value_t = 4)]
+    batches_per_msg: usize,
+
+    /// Number of consuming threads (number of threads requesting batches from scheduler)
+    #[clap(long, env, default_value_t = 20)]
+    num_execution_threads: usize,
+
+    /// How long each transaction takes to execution in microseconds
+    #[clap(long, env, default_value_t = 15)]
+    execution_per_tx_us: u64,
+
+    /// Duration of benchmark
+    #[clap(long, env, default_value_t = 20.0)]
+    duration: f32,
+
+    /// Number of accounts to choose from when signing transactions
+    #[clap(long, env, default_value_t = 100000)]
+    num_accounts: usize,
+
+    /// Number of read locks per tx
+    #[clap(long, env, default_value_t = 4)]
+    num_read_locks_per_tx: usize,
+
+    /// Number of write locks per tx
+    #[clap(long, env, default_value_t = 2)]
+    num_read_write_locks_per_tx: usize,
+
+    /// Max batch size for scheduler
+    #[clap(long, env, default_value_t = 128)]
+    max_batch_size: usize,
+
+    /// High-conflict sender
+    #[clap(long, env, default_value_t = 0)]
+    high_conflict_sender: usize,
+}
+
+/// Some convenient type aliases
+type TransactionMessage = Box<SanitizedTransaction>;
+type CompletedTransactionMessage = (usize, TransactionMessage); // thread index and transaction message
+type TransactionBatchMessage = Vec<TransactionMessage>;
+
+/// Dummy scheduler that you should replace with your own implementation
+struct TransactionScheduler;
+
+impl TransactionScheduler {
+    pub fn spawn_scheduler(
+        _packet_batch_receiver: Receiver<Vec<PacketBatch>>,
+        _transaction_batch_senders: Vec<Sender<TransactionBatchMessage>>,
+        _completed_transaction_receiver: Receiver<CompletedTransactionMessage>,
+        _bank_forks: Arc<RwLock<BankForks>>,
+        _max_batch_size: usize,
+        _exit: Arc<AtomicBool>,
+    ) -> JoinHandle<()> {
+        todo!()
+    }
+}
+
+#[derive(Debug, Default)]
+struct TransactionSchedulerBenchMetrics {
+    /// Number of transactions sent to the scheduler
+    num_transactions_sent: AtomicUsize,
+    /// Number of transactions scheduled
+    num_transactions_scheduled: AtomicUsize,
+    /// Number of transactions completed
+    num_transactions_completed: AtomicUsize,
+    /// Priority collected
+    priority_collected: AtomicU64,
+}
+
+impl TransactionSchedulerBenchMetrics {
+    fn report(&self) {
+        let num_transactions_sent = self.num_transactions_sent.load(Ordering::Relaxed);
+        let num_transactions_scheduled = self.num_transactions_scheduled.load(Ordering::Relaxed);
+        let num_transactions_completed = self.num_transactions_completed.load(Ordering::Relaxed);
+        let priority_collected = self.priority_collected.load(Ordering::Relaxed);
+
+        let num_transactions_pending = num_transactions_sent - num_transactions_scheduled;
+        info!("num_transactions_sent: {num_transactions_sent} num_transactions_pending: {num_transactions_pending} num_transactions_scheduled: {num_transactions_scheduled} num_transactions_completed: {num_transactions_completed} priority_collected: {priority_collected}");
+    }
+}
+
+struct PacketSendingConfig {
+    packets_per_batch: usize,
+    batches_per_msg: usize,
+    packet_send_rate: usize,
+    num_read_locks_per_tx: usize,
+    num_write_locks_per_tx: usize,
+}
+
+fn main() {
+    solana_logger::setup_with_default("INFO");
+
+    let Args {
+        packet_send_rate,
+        packets_per_batch,
+        batches_per_msg,
+        num_execution_threads,
+        execution_per_tx_us,
+        duration,
+        num_accounts,
+        num_read_locks_per_tx,
+        num_read_write_locks_per_tx,
+        max_batch_size,
+        high_conflict_sender,
+    } = Args::parse();
+
+    assert!(high_conflict_sender <= num_accounts);
+
+    let (packet_batch_sender, packet_batch_receiver) = crossbeam_channel::unbounded();
+    let (transaction_batch_senders, transaction_batch_receivers) =
+        build_channels(num_execution_threads);
+    let (completed_transaction_sender, completed_transaction_receiver) =
+        crossbeam_channel::unbounded();
+    let bank_forks = Arc::new(RwLock::new(BankForks::new(Bank::default_for_tests())));
+    let exit = Arc::new(AtomicBool::new(false));
+
+    // Spawns and runs the scheduler thread
+    let scheduler_handle = TransactionScheduler::spawn_scheduler(
+        packet_batch_receiver,
+        transaction_batch_senders,
+        completed_transaction_receiver,
+        bank_forks,
+        max_batch_size,
+        exit.clone(),
+    );
+
+    let metrics = Arc::new(TransactionSchedulerBenchMetrics::default());
+
+    // Spawn the execution threads (sleep on transactions and then send completed batches back)
+    let execution_handles = start_execution_threads(
+        metrics.clone(),
+        transaction_batch_receivers,
+        completed_transaction_sender,
+        execution_per_tx_us,
+        exit.clone(),
+    );
+
+    // Spawn thread to create and send packet batches
+    info!("building accounts...");
+    let accounts = Arc::new(build_accounts(num_accounts));
+    info!("built accounts...");
+    info!("starting packet senders...");
+    let duration = Duration::from_secs_f32(duration);
+    let packet_sending_config = Arc::new(PacketSendingConfig {
+        packets_per_batch,
+        batches_per_msg,
+        packet_send_rate,
+        num_read_locks_per_tx,
+        num_write_locks_per_tx: num_read_write_locks_per_tx,
+    });
+    let packet_sender_handles = spawn_packet_senders(
+        metrics.clone(),
+        high_conflict_sender,
+        accounts,
+        packet_batch_sender,
+        packet_sending_config,
+        duration,
+        exit.clone(),
+    );
+
+    // Spawn thread for reporting metrics
+    std::thread::spawn({
+        move || {
+            let start = Instant::now();
+            loop {
+                if exit.load(Ordering::Relaxed) {
+                    break;
+                }
+                if start.elapsed() > duration {
+                    let pending_transactions =
+                        metrics.num_transactions_sent.load(Ordering::Relaxed)
+                            - metrics.num_transactions_completed.load(Ordering::Relaxed);
+                    if pending_transactions == 0 {
+                        break;
+                    }
+                }
+
+                metrics.report();
+                std::thread::sleep(Duration::from_millis(100));
+            }
+            exit.store(true, Ordering::Relaxed);
+        }
+    });
+
+    scheduler_handle.join().unwrap();
+    execution_handles
+        .into_iter()
+        .for_each(|jh| jh.join().unwrap());
+    packet_sender_handles
+        .into_iter()
+        .for_each(|jh| jh.join().unwrap());
+}
+
+fn start_execution_threads(
+    metrics: Arc<TransactionSchedulerBenchMetrics>,
+    transaction_batch_receivers: Vec<Receiver<TransactionBatchMessage>>,
+    completed_transaction_sender: Sender<CompletedTransactionMessage>,
+    execution_per_tx_us: u64,
+    exit: Arc<AtomicBool>,
+) -> Vec<JoinHandle<()>> {
+    transaction_batch_receivers
+        .into_iter()
+        .enumerate()
+        .map(|(thread_index, transaction_batch_receiver)| {
+            start_execution_thread(
+                metrics.clone(),
+                thread_index,
+                transaction_batch_receiver,
+                completed_transaction_sender.clone(),
+                execution_per_tx_us,
+                exit.clone(),
+            )
+        })
+        .collect()
+}
+
+fn start_execution_thread(
+    metrics: Arc<TransactionSchedulerBenchMetrics>,
+    thread_index: usize,
+    transaction_batch_receiver: Receiver<TransactionBatchMessage>,
+    completed_transaction_sender: Sender<CompletedTransactionMessage>,
+    execution_per_tx_us: u64,
+    exit: Arc<AtomicBool>,
+) -> JoinHandle<()> {
+    std::thread::spawn(move || {
+        execution_worker(
+            metrics,
+            thread_index,
+            transaction_batch_receiver,
+            completed_transaction_sender,
+            execution_per_tx_us,
+            exit,
+        )
+    })
+}
+
+fn execution_worker(
+    metrics: Arc<TransactionSchedulerBenchMetrics>,
+    thread_index: usize,
+    transaction_batch_receiver: Receiver<TransactionBatchMessage>,
+    completed_transaction_sender: Sender<CompletedTransactionMessage>,
+    execution_per_tx_us: u64,
+    exit: Arc<AtomicBool>,
+) {
+    loop {
+        if exit.load(Ordering::Relaxed) {
+            break;
+        }
+
+        select! {
+            recv(transaction_batch_receiver) -> maybe_tx_batch => {
+                if let Ok(tx_batch) = maybe_tx_batch {
+                    handle_transaction_batch(&metrics, thread_index, &completed_transaction_sender, tx_batch, execution_per_tx_us);
+                }
+            }
+            default(Duration::from_millis(100)) => {}
+        }
+    }
+}
+
+fn handle_transaction_batch(
+    metrics: &TransactionSchedulerBenchMetrics,
+    thread_index: usize,
+    completed_transaction_sender: &Sender<CompletedTransactionMessage>,
+    transaction_batch: TransactionBatchMessage,
+    execution_per_tx_us: u64,
+) {
+    let num_transactions = transaction_batch.len() as u64;
+    metrics
+        .num_transactions_scheduled
+        .fetch_add(num_transactions as usize, Ordering::Relaxed);
+
+    sleep(Duration::from_micros(
+        num_transactions * execution_per_tx_us,
+    ));
+
+    let priority_collected = transaction_batch
+        .iter()
+        .map(|tx| tx.get_transaction_priority_details().unwrap().priority)
+        .sum();
+
+    metrics
+        .num_transactions_completed
+        .fetch_add(num_transactions as usize, Ordering::Relaxed);
+    metrics
+        .priority_collected
+        .fetch_add(priority_collected, Ordering::Relaxed);
+
+    for transaction in transaction_batch {
+        completed_transaction_sender
+            .send((thread_index, transaction))
+            .unwrap();
+    }
+}
+
+const NUM_SENDERS: usize = 4;
+
+fn spawn_packet_senders(
+    metrics: Arc<TransactionSchedulerBenchMetrics>,
+    high_conflict_sender: usize,
+    accounts: Arc<Vec<Keypair>>,
+    packet_batch_sender: Sender<Vec<PacketBatch>>,
+    config: Arc<PacketSendingConfig>,
+    duration: Duration,
+    exit: Arc<AtomicBool>,
+) -> Vec<JoinHandle<()>> {
+    (0..NUM_SENDERS)
+        .map(|i| {
+            let num_accounts = if i == 0 && high_conflict_sender > 0 {
+                high_conflict_sender
+            } else {
+                accounts.len()
+            };
+            spawn_packet_sender(
+                metrics.clone(),
+                num_accounts,
+                accounts.clone(),
+                packet_batch_sender.clone(),
+                config.clone(),
+                duration,
+                exit.clone(),
+            )
+        })
+        .collect()
+}
+
+fn spawn_packet_sender(
+    metrics: Arc<TransactionSchedulerBenchMetrics>,
+    num_accounts: usize,
+    accounts: Arc<Vec<Keypair>>,
+    packet_batch_sender: Sender<Vec<PacketBatch>>,
+    config: Arc<PacketSendingConfig>,
+    duration: Duration,
+    exit: Arc<AtomicBool>,
+) -> JoinHandle<()> {
+    std::thread::spawn(move || {
+        send_packets(
+            metrics,
+            num_accounts,
+            accounts,
+            packet_batch_sender,
+            config,
+            duration,
+            exit,
+        );
+    })
+}
+
+fn send_packets(
+    metrics: Arc<TransactionSchedulerBenchMetrics>,
+    num_accounts: usize,
+    accounts: Arc<Vec<Keypair>>,
+    packet_batch_sender: Sender<Vec<PacketBatch>>,
+    config: Arc<PacketSendingConfig>,
+    duration: Duration,
+    exit: Arc<AtomicBool>,
+) {
+    let packets_per_msg = config.packets_per_batch * config.batches_per_msg;
+    let loop_frequency =
+        config.packet_send_rate as f64 * packets_per_msg as f64 / NUM_SENDERS as f64;
+    let loop_duration = Duration::from_secs_f64(1.0 / loop_frequency);
+
+    info!("sending packets: packets_per_msg: {packets_per_msg} loop_frequency: {loop_frequency} loop_duration: {loop_duration:?}");
+
+    let blockhash = Hash::default();
+    let start = Instant::now();
+
+    loop {
+        if exit.load(Ordering::Relaxed) {
+            break;
+        }
+        if start.elapsed() > duration {
+            info!("stopping packet sending");
+            break;
+        }
+        let (packet_batches, packet_build_time) = measure!(build_packet_batches(
+            &config,
+            num_accounts,
+            &accounts,
+            &blockhash
+        ));
+        metrics.num_transactions_sent.fetch_add(
+            packet_batches.iter().map(|pb| pb.len()).sum(),
+            Ordering::Relaxed,
+        );
+        let _ = packet_batch_sender.send(packet_batches);
+
+        std::thread::sleep(loop_duration.saturating_sub(packet_build_time.as_duration()));
+    }
+}
+
+fn build_packet_batches(
+    config: &PacketSendingConfig,
+    num_accounts: usize,
+    accounts: &[Keypair],
+    blockhash: &Hash,
+) -> Vec<PacketBatch> {
+    (0..config.batches_per_msg)
+        .map(|_| build_packet_batch(config, num_accounts, accounts, blockhash))
+        .collect()
+}
+
+fn build_packet_batch(
+    config: &PacketSendingConfig,
+    num_accounts: usize,
+    accounts: &[Keypair],
+    blockhash: &Hash,
+) -> PacketBatch {
+    PacketBatch::new(
+        (0..config.packets_per_batch)
+            .map(|_| build_packet(config, num_accounts, accounts, blockhash))
+            .collect(),
+    )
+}
+
+fn build_packet(
+    config: &PacketSendingConfig,
+    num_accounts: usize,
+    accounts: &[Keypair],
+    blockhash: &Hash,
+) -> Packet {
+    let get_random_account = || &accounts[rand::thread_rng().gen_range(0..num_accounts)];
+    let sending_keypair = get_random_account();
+
+    let read_account_metas = (0..config.num_read_locks_per_tx)
+        .map(|_| AccountMeta::new_readonly(get_random_account().pubkey(), false));
+    let write_account_metas = (0..config.num_write_locks_per_tx)
+        .map(|_| AccountMeta::new(get_random_account().pubkey(), false));
+    let ixs = vec![
+        ComputeBudgetInstruction::set_compute_unit_price(rand::thread_rng().gen_range(50..500)),
+        Instruction::new_with_bytes(
+            system_program::id(),
+            &[0],
+            read_account_metas.chain(write_account_metas).collect(),
+        ),
+    ];
+    let versioned_transaction = VersionedTransaction::from(Transaction::new_signed_with_payer(
+        &ixs,
+        Some(&sending_keypair.pubkey()),
+        &[sending_keypair],
+        *blockhash,
+    ));
+    Packet::from_data(None, &versioned_transaction).unwrap()
+}
+
+fn build_accounts(num_accounts: usize) -> Vec<Keypair> {
+    (0..num_accounts).map(|_| Keypair::new()).collect()
+}
+
+fn build_channels<T>(num_execution_threads: usize) -> (Vec<Sender<T>>, Vec<Receiver<T>>) {
+    let mut senders = Vec::with_capacity(num_execution_threads);
+    let mut receivers = Vec::with_capacity(num_execution_threads);
+    for _ in 0..num_execution_threads {
+        let (sender, receiver) = crossbeam_channel::unbounded();
+        senders.push(sender);
+        receivers.push(receiver);
+    }
+    (senders, receivers)
+}

From 28587d9abe2722492751034c21309bd4bee08f97 Mon Sep 17 00:00:00 2001
From: Andrew Fitzgerald <apfitzge@gmail.com>
Date: Tue, 2 Aug 2022 13:32:52 -0500
Subject: [PATCH 0002/3199] transaction scheduler

---
 core/src/lib.rs                   |   1 +
 core/src/transaction_scheduler.rs | 988 ++++++++++++++++++++++++++++++
 2 files changed, 989 insertions(+)
 create mode 100644 core/src/transaction_scheduler.rs

diff --git a/core/src/lib.rs b/core/src/lib.rs
index 9be6ab5b9cc4ba..10340cb72e7505 100644
--- a/core/src/lib.rs
+++ b/core/src/lib.rs
@@ -69,6 +69,7 @@ pub mod tower_storage;
 pub mod tpu;
 pub mod tracer_packet_stats;
 pub mod transaction_priority_details;
+pub mod transaction_scheduler;
 pub mod tree_diff;
 pub mod tvu;
 pub mod unfrozen_gossip_verified_vote_hashes;
diff --git a/core/src/transaction_scheduler.rs b/core/src/transaction_scheduler.rs
new file mode 100644
index 00000000000000..fed4c2edb6ba75
--- /dev/null
+++ b/core/src/transaction_scheduler.rs
@@ -0,0 +1,988 @@
+//! Implements a transaction scheduler
+
+use {
+    crate::{
+        transaction_priority_details::GetTransactionPriorityDetails,
+        unprocessed_packet_batches::{self, ImmutableDeserializedPacket},
+    },
+    crossbeam_channel::{Receiver, Sender, TryRecvError},
+    dashmap::DashMap,
+    solana_measure::{measure, measure::Measure},
+    solana_perf::packet::PacketBatch,
+    solana_runtime::{bank::Bank, bank_forks::BankForks},
+    solana_sdk::{
+        pubkey::Pubkey,
+        transaction::{SanitizedTransaction, TransactionError},
+    },
+    std::{
+        collections::{BinaryHeap, HashMap, HashSet, VecDeque},
+        hash::Hash,
+        sync::{
+            atomic::{AtomicBool, Ordering},
+            Arc, Mutex, RwLock,
+        },
+        thread::JoinHandle,
+        time::Instant,
+    },
+};
+/// Wrapper to store a sanitized transaction and priority
+#[derive(Debug)]
+pub struct TransactionPriority {
+    /// Transaction priority
+    pub priority: u64,
+    /// Sanitized transaction
+    pub transaction: Box<SanitizedTransaction>,
+    /// Timestamp the scheduler received the transaction - only used for ordering
+    pub timestamp: Instant,
+}
+
+impl Ord for TransactionPriority {
+    fn cmp(&self, other: &Self) -> std::cmp::Ordering {
+        match self.priority.cmp(&other.priority) {
+            std::cmp::Ordering::Equal => match self
+                .transaction
+                .message_hash()
+                .cmp(other.transaction.message_hash())
+            {
+                std::cmp::Ordering::Equal => self.timestamp.cmp(&other.timestamp),
+                ordering => ordering,
+            },
+            ordering => ordering,
+        }
+    }
+}
+
+impl PartialOrd for TransactionPriority {
+    fn partial_cmp(&self, other: &Self) -> Option<std::cmp::Ordering> {
+        Some(self.cmp(other))
+    }
+}
+
+impl PartialEq for TransactionPriority {
+    fn eq(&self, other: &Self) -> bool {
+        self.priority == other.priority
+            && self.transaction.message_hash() == other.transaction.message_hash()
+            && self.timestamp == other.timestamp
+    }
+}
+
+impl Eq for TransactionPriority {}
+
+impl Hash for TransactionPriority {
+    fn hash<H: std::hash::Hasher>(&self, state: &mut H) {
+        self.transaction.signature().hash(state);
+        self.timestamp.hash(state);
+        self.priority.hash(state);
+    }
+}
+
+type TransactionRef = TransactionPriority;
+
+impl TransactionPriority {
+    fn new(transaction: SanitizedTransaction, num_conflicts: u64) -> TransactionRef {
+        let packet_priority = transaction
+            .get_transaction_priority_details()
+            .unwrap()
+            .priority;
+        let priority = Self::modify_priority(packet_priority, num_conflicts);
+
+        Self {
+            transaction: Box::new(transaction),
+            priority,
+            timestamp: Instant::now(),
+        }
+    }
+
+    fn build_and_verify_transaction(
+        packet: &ImmutableDeserializedPacket,
+        bank: &Bank,
+    ) -> Result<SanitizedTransaction, TransactionError> {
+        let transaction = SanitizedTransaction::try_new(
+            packet.transaction().clone(),
+            *packet.message_hash(),
+            packet.is_simple_vote(),
+            bank,
+        )?;
+        transaction.verify_precompiles(&bank.feature_set)?;
+        Ok(transaction)
+    }
+
+    fn modify_priority(packet_priority: u64, _num_conflicts: u64) -> u64 {
+        // (packet_priority * 1_000_000_000) / (1 + num_conflicts)
+        packet_priority
+    }
+}
+
+type PacketBatchMessage = Vec<PacketBatch>;
+type TransactionMessage = Box<SanitizedTransaction>;
+type CompletedTransactionMessage = (usize, TransactionMessage);
+type TransactionBatchMessage = Vec<TransactionMessage>;
+
+/// Separate packet deserialization and ordering
+struct PacketBatchHandler {
+    /// Exit signal
+    exit: Arc<AtomicBool>,
+    /// Bank
+    bank_forks: Arc<RwLock<BankForks>>,
+    /// Channel for receiving deserialized packet batches from SigVerify
+    packet_batch_receiver: Receiver<PacketBatchMessage>,
+    /// Pending transactions to be send to the scheduler
+    pending_transactions: Arc<Mutex<BinaryHeap<TransactionRef>>>,
+    /// Account Queues
+    transactions_by_account: Arc<DashMap<Pubkey, AccountTransactionQueue>>,
+}
+
+impl PacketBatchHandler {
+    /// Driving loop
+    fn main(mut self) {
+        loop {
+            if self.exit.load(Ordering::Relaxed) {
+                break;
+            }
+            self.iter();
+        }
+    }
+
+    /// Try receiving packets or send out buffered transactions
+    fn iter(&mut self) {
+        if let Ok(packet_batches) = self.packet_batch_receiver.try_recv() {
+            self.handle_packet_batches(packet_batches);
+        }
+    }
+
+    /// Handle received packet batches - deserialize and put into the buffer
+    fn handle_packet_batches(&mut self, packet_batches: Vec<PacketBatch>) {
+        let bank = self.bank_forks.read().unwrap().working_bank();
+        for packet_batch in packet_batches {
+            let packet_indices = packet_batch
+                .into_iter()
+                .enumerate()
+                .filter_map(|(idx, p)| if !p.meta.discard() { Some(idx) } else { None })
+                .collect::<Vec<_>>();
+            let transactions =
+                unprocessed_packet_batches::deserialize_packets(&packet_batch, &packet_indices)
+                    .filter_map(|deserialized_packet| {
+                        TransactionPriority::build_and_verify_transaction(
+                            deserialized_packet.immutable_section(),
+                            &bank,
+                        )
+                        .ok()
+                    })
+                    .map(|transaction| {
+                        let num_conflicts = self.get_num_conflicts(&transaction);
+                        TransactionPriority::new(transaction, num_conflicts)
+                    })
+                    .collect::<Vec<_>>();
+            self.insert_transactions(transactions);
+        }
+    }
+
+    /// Count conflicts
+    fn get_num_conflicts(&self, transaction: &SanitizedTransaction) -> u64 {
+        let account_locks = transaction.get_account_locks().unwrap();
+
+        // let mut read_conflicts = 0;
+        for account in account_locks.readonly.into_iter() {
+            let mut queue = self.transactions_by_account.entry(*account).or_default();
+            // read_conflicts += queue.writes;
+            queue.reads += 1;
+        }
+
+        let mut write_conflicts = 0;
+        for account in account_locks.writable.into_iter() {
+            let mut queue = self.transactions_by_account.entry(*account).or_default();
+            write_conflicts += queue.writes + queue.reads;
+            queue.writes += 1;
+        }
+
+        // read_conflicts + write_conflicts
+        write_conflicts
+    }
+
+    /// Insert transactions into queues and pending
+    fn insert_transactions(&self, transactions: Vec<TransactionRef>) {
+        for tx in &transactions {
+            // Get account locks
+            let account_locks = tx.transaction.get_account_locks().unwrap();
+            for account in account_locks.readonly.into_iter() {
+                self.transactions_by_account.entry(*account).or_default();
+                // .reads
+                // .insert(tx.clone());
+            }
+
+            for account in account_locks.writable.into_iter() {
+                self.transactions_by_account.entry(*account).or_default();
+                // .writes
+                // .insert(tx.clone());
+            }
+        }
+        self.pending_transactions
+            .lock()
+            .unwrap()
+            .extend(transactions.into_iter());
+    }
+}
+
+/// Stores state for scheduling transactions and channels for communicating
+/// with other threads: SigVerify and Banking
+pub struct TransactionScheduler {
+    /// Channels for sending transaction batches to banking threads
+    transaction_batch_senders: Vec<Sender<TransactionBatchMessage>>,
+    /// Channel for receiving completed transactions from any banking thread
+    completed_transaction_receiver: Receiver<CompletedTransactionMessage>,
+    /// Max number of transactions to send to a single banking-thread in a batch
+    max_batch_size: usize,
+    /// Exit signal
+    exit: Arc<AtomicBool>,
+
+    /// Pending transactions that are not known to be blocked
+    pending_transactions: Arc<Mutex<BinaryHeap<TransactionRef>>>,
+    /// Transaction queues and locks by account key
+    transactions_by_account: Arc<DashMap<Pubkey, AccountTransactionQueue>>,
+    /// Map from transaction signature to transactions blocked by the signature
+    // blocked_transactions: HashMap<Signature, Vec<TransactionRef>>,
+    /// Map from blocking BatchId to transactions
+    blocked_transactions_by_batch_id: HashMap<TransactionBatchId, HashSet<TransactionRef>>,
+    /// Transactions blocked by batches need to count how many they're blocked by
+    // blocked_transactions_batch_count: HashMap<TransactionRef, usize>,
+    /// Tracks the current number of blocked transactions
+    num_blocked_transactions: usize,
+    /// Tracks the current number of executing transacitons
+    num_executing_transactions: usize,
+
+    /// Generates TransactionBatchIds
+    next_transaction_batch_id: TransactionBatchId,
+    /// Tracks TransactionBatchDetails by TransactionBatchId
+    transaction_batches: HashMap<TransactionBatchId, TransactionBatch>,
+    /// Currently in-progress batches (references into `transaction_batches`)
+    in_progress_batches: HashSet<TransactionBatchId>,
+    oldest_in_progress_batch: Option<TransactionBatchId>,
+
+    /// Tracks status of exeuction threads
+    execution_thread_stats: Vec<ExecutionThreadStats>,
+
+    /// Track metrics for scheduler thread
+    metrics: SchedulerMetrics,
+}
+
+impl TransactionScheduler {
+    /// Create and start transaction scheduler thread
+    pub fn spawn_scheduler(
+        packet_batch_receiver: Receiver<PacketBatchMessage>,
+        transaction_batch_senders: Vec<Sender<TransactionBatchMessage>>,
+        completed_transaction_receiver: Receiver<CompletedTransactionMessage>,
+        bank_forks: Arc<RwLock<BankForks>>,
+        max_batch_size: usize,
+        exit: Arc<AtomicBool>,
+    ) -> JoinHandle<()> {
+        let pending_transactions = Arc::new(Mutex::new(BinaryHeap::default()));
+        let transactions_by_account = Arc::new(DashMap::default());
+
+        let packet_handler = PacketBatchHandler {
+            exit: exit.clone(),
+            bank_forks,
+            packet_batch_receiver,
+            pending_transactions: pending_transactions.clone(),
+            transactions_by_account: transactions_by_account.clone(),
+        };
+
+        std::thread::spawn(move || packet_handler.main());
+        let num_execution_threads = transaction_batch_senders.len();
+        let execution_thread_stats = (0..num_execution_threads)
+            .into_iter()
+            .map(|_| ExecutionThreadStats::default())
+            .collect();
+
+        let mut scheduler = TransactionScheduler {
+            transaction_batch_senders,
+            completed_transaction_receiver,
+            max_batch_size,
+            exit,
+            pending_transactions,
+            transactions_by_account,
+            // blocked_transactions: HashMap::default(),
+            blocked_transactions_by_batch_id: HashMap::default(),
+            // blocked_transactions_batch_count: HashMap::default(),
+            num_blocked_transactions: 0,
+            num_executing_transactions: 0,
+            next_transaction_batch_id: 0,
+            transaction_batches: HashMap::default(),
+            in_progress_batches: HashSet::with_capacity(num_execution_threads),
+            oldest_in_progress_batch: None,
+            execution_thread_stats,
+            metrics: SchedulerMetrics::default(),
+        };
+
+        // Initialize batch for thread 0
+        scheduler.create_new_batch();
+
+        std::thread::spawn(move || scheduler.main())
+    }
+
+    /// Driving loop
+    fn main(mut self) {
+        loop {
+            if self.exit.load(Ordering::Relaxed) {
+                break;
+            }
+            self.iter();
+        }
+    }
+
+    /// Performs work in a loop - Handles different channel receives/timers and performs scheduling
+    fn iter(&mut self) {
+        fn try_recv<T>(receiver: &Receiver<T>) -> (Result<T, TryRecvError>, Measure) {
+            measure!(receiver.try_recv())
+        }
+
+        // Try receiving completed batches
+        let (_, completed_transaction_time) = measure!({
+            let (maybe_completed_transaction, recv_time) =
+                try_recv(&self.completed_transaction_receiver);
+            let (_, handle_batch_time) = measure!({
+                if let Ok((thread_index, completed_transaction)) = maybe_completed_transaction {
+                    self.handle_completed_transaction(thread_index, completed_transaction);
+                }
+            });
+
+            self.metrics.compeleted_batch_try_recv_time_us += recv_time.as_us();
+            self.metrics.completed_batch_handle_batch_time_us += handle_batch_time.as_us();
+        });
+        self.metrics.completed_transactions_time_us += completed_transaction_time.as_us();
+
+        // Scheduling time
+        let (_, scheduling_time) = measure!(self.do_scheduling());
+        self.metrics.scheduling_time_us += scheduling_time.as_us();
+
+        // Check if oldest batch should be sent due to age
+        if let Some(oldest_batch_id) = self.oldest_in_progress_batch {
+            let batch = self.transaction_batches.get(&oldest_batch_id).unwrap();
+            if batch.start_time.elapsed() >= std::time::Duration::from_millis(1000) {
+                self.send_batch(oldest_batch_id);
+            }
+        }
+
+        let (_, metrics_time) = measure!(self.metrics.report());
+        self.metrics.metrics_us += metrics_time.as_us();
+    }
+
+    /// Handle completed transaction batch
+    fn handle_completed_transaction(
+        &mut self,
+        execution_thread_index: usize,
+        _transaction: TransactionMessage,
+    ) {
+        // TODO: verify transaction signature matches expectation?
+        // Update queued stats
+        self.num_executing_transactions -= 1;
+        let execution_thread_stats = &mut self.execution_thread_stats[execution_thread_index];
+        // TODO: actually track CUs instead of just num transactions
+        execution_thread_stats.queued_compute_units -= 1;
+        execution_thread_stats.queued_transactions -= 1;
+
+        // Check if the batch is complete
+        let batch_id = *execution_thread_stats
+            .queued_batches
+            .front()
+            .expect("should not receive compelted transaction without a queued batch");
+        let batch = self
+            .transaction_batches
+            .get_mut(&batch_id)
+            .expect("queued batch id should exist");
+        batch.num_transactions = batch.num_transactions.saturating_sub(1);
+
+        if batch.num_transactions == 0 {
+            // Remove the batch
+            let batch = self.transaction_batches.remove(&batch_id).unwrap();
+            execution_thread_stats.queued_batches.pop_front().unwrap();
+
+            // Remove account locks
+            let (_, remove_account_locks_time) = measure!({
+                for (account, _lock) in batch.account_locks {
+                    self.unlock_account(account, batch_id);
+                }
+            });
+
+            // Push transactions blocked (by batch) back into the pending queue
+            let (_, unblock_transactions_time) = measure!({
+                if let Some(blocked_transactions) =
+                    self.blocked_transactions_by_batch_id.remove(&batch_id)
+                {
+                    self.num_blocked_transactions -= blocked_transactions.len();
+                    self.pending_transactions
+                        .lock()
+                        .unwrap()
+                        .extend(blocked_transactions.into_iter());
+                }
+            });
+
+            // Create a new batch for the thread (if necessary)
+            self.create_new_batch();
+
+            self.metrics.completed_transactions_remove_account_locks_us +=
+                remove_account_locks_time.as_us();
+            self.metrics.completed_transactions_unblock_transactions_us +=
+                unblock_transactions_time.as_us();
+        }
+    }
+
+    /// Remove account locks for the batch id
+    fn unlock_account(&mut self, account: Pubkey, batch_id: TransactionBatchId) {
+        self.transactions_by_account
+            .get_mut(&account)
+            .unwrap()
+            .unlock(batch_id)
+    }
+
+    fn create_new_batch(&mut self) {
+        // Find first thread with no work on it and NO threads before that have an in-progress
+        let mut execution_thread_index = None;
+        for (thread_index, execution_thread_stats) in self.execution_thread_stats.iter().enumerate()
+        {
+            let num_queued_batches = execution_thread_stats.queued_batches.len();
+            // If no scheduled work on this thread, create the new batch here
+            if num_queued_batches == 0 {
+                execution_thread_index = Some(thread_index);
+                break;
+            } else if num_queued_batches == 1 {
+                if self
+                    .in_progress_batches
+                    .contains(execution_thread_stats.queued_batches.back().unwrap())
+                {
+                    return; // don't create a new batch since we already have one that's not been filled up yet
+                } else {
+                    // This thread has a free-queue spot, create a new batch here
+                    execution_thread_index = Some(thread_index);
+                    break;
+                }
+            }
+        }
+
+        if let Some(execution_thread_index) = execution_thread_index {
+            self.metrics.max_thread_index_used = self
+                .metrics
+                .max_thread_index_used
+                .max(execution_thread_index);
+
+            let batch_id = self.next_transaction_batch_id;
+            self.next_transaction_batch_id += 1;
+            self.transaction_batches.insert(
+                batch_id,
+                TransactionBatch {
+                    scheduled: false,
+                    start_time: Instant::now(),
+                    id: batch_id,
+                    num_transactions: 0,
+                    transactions: Vec::with_capacity(self.max_batch_size),
+                    account_locks: HashMap::default(),
+                    execution_thread_index,
+                },
+            );
+            self.execution_thread_stats[execution_thread_index]
+                .queued_batches
+                .push_back(batch_id);
+            self.in_progress_batches.insert(batch_id);
+            self.oldest_in_progress_batch.get_or_insert(batch_id);
+        }
+    }
+
+    /// Send an in-progress batch
+    fn send_batch(&mut self, batch_id: usize) {
+        let batch = self.transaction_batches.get_mut(&batch_id).unwrap();
+        self.metrics.max_batch_age = self
+            .metrics
+            .max_batch_age
+            .max(batch.start_time.elapsed().as_micros() as u64);
+        let execution_thread_index = batch.execution_thread_index;
+
+        if !batch.transactions.is_empty() {
+            // 1. Send the batch to thread
+            // Build the batch
+            let transactions = batch.build_account_locks_on_send();
+
+            // Update execution thread stats
+            let execution_thread_stats = &mut self.execution_thread_stats[execution_thread_index];
+            execution_thread_stats.queued_transactions += transactions.len();
+
+            // Update metrics
+            self.num_executing_transactions += transactions.len();
+            self.metrics.num_transactions_scheduled += transactions.len();
+            self.metrics.max_blocked_transactions = self
+                .metrics
+                .max_blocked_transactions
+                .max(self.num_blocked_transactions);
+            self.metrics.max_executing_transactions = self
+                .num_executing_transactions
+                .max(self.num_executing_transactions);
+
+            // Send the batch
+            self.transaction_batch_senders[execution_thread_index]
+                .send(transactions)
+                .unwrap();
+
+            // 2. Remove from in-progress batch set
+            self.in_progress_batches.remove(&batch_id);
+
+            // 3. Check if we should create a new batch for the thread
+            self.create_new_batch();
+            if batch_id == self.oldest_in_progress_batch.unwrap() {
+                self.oldest_in_progress_batch = self.in_progress_batches.iter().min().cloned()
+            }
+            // 4. Possibly update the oldest batch
+        } else {
+            self.execution_thread_stats[execution_thread_index]
+                .queued_batches
+                .pop_front()
+                .unwrap();
+            self.transaction_batches.remove(&batch_id).unwrap();
+            self.in_progress_batches.remove(&batch_id);
+            self.create_new_batch();
+            if batch_id == self.oldest_in_progress_batch.unwrap() {
+                self.oldest_in_progress_batch = self.in_progress_batches.iter().min().cloned()
+            }
+        }
+    }
+
+    /// Performs scheduling operations on currently pending transactions
+    fn do_scheduling(&mut self) {
+        if self.in_progress_batches.is_empty() {
+            return;
+        }
+        let num_pending_transactions = self.pending_transactions.lock().unwrap().len();
+        self.metrics.max_pending_transactions = self
+            .metrics
+            .max_pending_transactions
+            .max(num_pending_transactions);
+        let maybe_transaction = self.pending_transactions.lock().unwrap().pop();
+        if let Some(transaction) = maybe_transaction {
+            self.try_schedule_transaction(transaction);
+        }
+    }
+
+    /// Try to schedule a transaction
+    fn try_schedule_transaction(&mut self, transaction: TransactionRef) {
+        // Check for blocking transactions batches in scheduled locks (this includes batches currently being built)
+        let (conflicting_batches, get_conflicting_batches_time) =
+            measure!(self.get_conflicting_batches(&transaction));
+        self.metrics.get_conflicting_batches_time += get_conflicting_batches_time.as_us();
+
+        let maybe_batch_id = if let Some(conflicting_batches) = conflicting_batches.as_ref() {
+            let mut schedulable_thread_index = None;
+            for batch_id in conflicting_batches {
+                let thread_index = self
+                    .transaction_batches
+                    .get(batch_id)
+                    .unwrap()
+                    .execution_thread_index;
+
+                if thread_index != *schedulable_thread_index.get_or_insert(thread_index) {
+                    schedulable_thread_index = None;
+                    break;
+                }
+            }
+            schedulable_thread_index
+                .map(|thread_index| {
+                    self.execution_thread_stats[thread_index]
+                        .queued_batches
+                        .back()
+                        .unwrap()
+                })
+                .filter(|batch_id| self.in_progress_batches.contains(*batch_id))
+                .cloned()
+        } else {
+            // Find the lowest-thread in-progress batch
+            self.in_progress_batches
+                .iter()
+                .map(|x| {
+                    (
+                        self.transaction_batches
+                            .get(x)
+                            .unwrap()
+                            .execution_thread_index,
+                        *x,
+                    )
+                })
+                .min()
+                .map(|(_, x)| x)
+        };
+
+        if maybe_batch_id.is_none() {
+            if let Some(conflicting_batches) = conflicting_batches {
+                self.num_blocked_transactions += 1;
+                let newest_conflicting_batch = conflicting_batches.into_iter().max().unwrap();
+                assert!(self
+                    .blocked_transactions_by_batch_id
+                    .entry(newest_conflicting_batch)
+                    .or_default()
+                    .insert(transaction));
+            }
+            return;
+        }
+
+        let batch_id = maybe_batch_id.unwrap();
+        // Schedule the transaction:
+        let (_, batching_time) = measure!({
+            // 1. Add account locks with the batch id
+            self.lock_accounts_for_transaction(&transaction, batch_id);
+            // 2. Add to Batch
+            let batch = self.transaction_batches.get_mut(&batch_id).unwrap();
+            assert!(!batch.scheduled);
+            batch.transactions.push(transaction);
+            // 3. Update queued execution stats
+            self.execution_thread_stats[batch.execution_thread_index].queued_compute_units += 1; // TODO: actually use CU instead of # tx
+            self.execution_thread_stats[batch.execution_thread_index].queued_transactions += 1;
+
+            // Check if batch should be sent
+            if batch.transactions.len() == self.max_batch_size {
+                let batch_id = batch.id;
+                self.send_batch(batch_id);
+            }
+        });
+        self.metrics.batching_time += batching_time.as_us();
+    }
+
+    /// Gets batches that conflict with the current transaction
+    ///     - Conflict does not necessarily mean block as they can be scheduled on the same thread
+    fn get_conflicting_batches(
+        &self,
+        transaction: &TransactionRef,
+    ) -> Option<HashSet<TransactionBatchId>> {
+        let mut conflicting_batches = HashSet::default();
+
+        let account_locks = transaction.transaction.get_account_locks().unwrap();
+
+        // Read accounts will only be blocked by writes on other threads
+        for account in account_locks.readonly.into_iter() {
+            for batch_id in self
+                .transactions_by_account
+                .get(account)
+                .unwrap()
+                .scheduled_lock
+                .write_batches
+                .iter()
+            {
+                conflicting_batches.insert(*batch_id);
+            }
+        }
+
+        // Write accounts will be blocked by reads or writes on other threads
+        for account in account_locks.writable.into_iter() {
+            let scheduled_lock = &self
+                .transactions_by_account
+                .get(account)
+                .unwrap()
+                .scheduled_lock;
+            for batch_id in scheduled_lock.write_batches.iter() {
+                conflicting_batches.insert(*batch_id);
+            }
+            for batch_id in scheduled_lock.read_batches.iter() {
+                conflicting_batches.insert(*batch_id);
+            }
+        }
+
+        (!conflicting_batches.is_empty()).then(|| conflicting_batches)
+    }
+
+    /// Lock accounts for a transaction by batch id
+    fn lock_accounts_for_transaction(
+        &mut self,
+        transaction: &TransactionRef,
+        batch_id: TransactionBatchId,
+    ) {
+        let accounts = transaction.transaction.get_account_locks().unwrap();
+        for account in accounts.readonly {
+            let mut queue = self.transactions_by_account.get_mut(account).unwrap();
+            queue.scheduled_lock.lock_on_batch(batch_id, false);
+            queue.reads -= 1;
+        }
+        for account in accounts.writable {
+            let mut queue = self.transactions_by_account.get_mut(account).unwrap();
+            queue.scheduled_lock.lock_on_batch(batch_id, true);
+            queue.writes -= 1;
+        }
+    }
+}
+
+/// Tracks all pending and blocked transacitons, ordered by priority, for a single account
+#[derive(Default)]
+struct AccountTransactionQueue {
+    reads: u64,  // unscheduled number of transactions reading
+    writes: u64, // unscheduled number of transactions writing
+
+    // /// Tree of read transactions on the account ordered by fee-priority
+    // reads: BTreeSet<TransactionRef>,
+    // /// Tree of write transactions on the account ordered by fee-priority
+    // writes: BTreeSet<TransactionRef>,
+    /// Tracks currently scheduled transactions on the account
+    scheduled_lock: AccountLock,
+}
+
+impl AccountTransactionQueue {
+    /// Unlocks the account queue for `batch_id`
+    fn unlock(&mut self, batch_id: TransactionBatchId) {
+        self.scheduled_lock.unlock_on_batch(batch_id);
+    }
+
+    // /// Find the minimum-priority transaction that blocks this transaction if there is one
+    // fn get_min_blocking_transaction(
+    //     &self,
+    //     transaction: &TransactionRef,
+    //     is_write: bool,
+    // ) -> Option<TransactionRef> {
+    //     let mut min_blocking_transaction = None;
+    //     // Write transactions will be blocked by higher-priority reads, but read transactions will not
+    //     if is_write {
+    //         min_blocking_transaction = option_min(
+    //             min_blocking_transaction,
+    //             upper_bound(&self.reads, transaction.clone()),
+    //         );
+    //     }
+
+    //     // All transactions are blocked by higher-priority write-transactions
+    //     option_min(
+    //         min_blocking_transaction,
+    //         upper_bound(&self.writes, transaction.clone()),
+    //     )
+    //     .map(|txr| txr.clone())
+    // }
+}
+
+/// Tracks the lock status of an account by batch id
+#[derive(Debug, Default)]
+struct AccountLock {
+    read_batches: HashSet<TransactionBatchId>,
+    write_batches: HashSet<TransactionBatchId>,
+}
+
+impl AccountLock {
+    fn lock_on_batch(&mut self, batch_id: TransactionBatchId, is_write: bool) {
+        if is_write {
+            // override read lock
+            self.read_batches.remove(&batch_id);
+            self.write_batches.insert(batch_id);
+        } else {
+            // underride write lock
+            if !self.write_batches.contains(&batch_id) {
+                self.read_batches.insert(batch_id);
+            }
+        }
+    }
+
+    fn unlock_on_batch(&mut self, batch_id: TransactionBatchId) {
+        self.read_batches.remove(&batch_id);
+        self.write_batches.remove(&batch_id);
+    }
+}
+
+#[derive(Debug, Clone)]
+enum AccountLockKind {
+    Read,
+    Write,
+}
+
+/// Identified for TransactionBatches
+pub type TransactionBatchId = usize;
+
+/// Transactions in a batch
+#[derive(Debug)]
+struct TransactionBatch {
+    /// Has the transaction been sent
+    scheduled: bool,
+    /// Timestamp of the batch starting to be built
+    start_time: Instant,
+    /// Identifier
+    id: TransactionBatchId,
+    /// Number of transactions
+    num_transactions: usize,
+    /// Transactions (only valid before send)
+    transactions: Vec<TransactionRef>,
+    /// Locked Accounts and Kind Set (only built on send)
+    account_locks: HashMap<Pubkey, AccountLockKind>,
+    /// Thread it is scheduled on
+    execution_thread_index: usize,
+}
+
+impl TransactionBatch {
+    fn build_account_locks_on_send(&mut self) -> TransactionBatchMessage {
+        self.scheduled = true;
+        self.num_transactions = self.transactions.len();
+        for transaction in self.transactions.iter() {
+            let account_locks = transaction.transaction.get_account_locks().unwrap();
+
+            for account in account_locks.readonly.into_iter() {
+                self.account_locks
+                    .entry(*account)
+                    .or_insert(AccountLockKind::Read);
+            }
+            for account in account_locks.writable.into_iter() {
+                self.account_locks.insert(*account, AccountLockKind::Write);
+            }
+        }
+
+        let mut transactions = Vec::new();
+        std::mem::swap(&mut transactions, &mut self.transactions);
+
+        transactions.into_iter().map(|tx| tx.transaction).collect()
+    }
+}
+
+/// Track stats for the execution threads - Order of members matters for derived implementations of PartialCmp
+#[derive(Debug, Default, Eq, Ord, PartialEq, PartialOrd)]
+struct ExecutionThreadStats {
+    /// Currently queued compute-units
+    queued_compute_units: usize,
+    /// Currently queued number of transactions
+    queued_transactions: usize,
+    /// Currently queued batch ids
+    queued_batches: VecDeque<TransactionBatchId>,
+}
+
+/// Track metrics for the scheduler thread
+struct SchedulerMetrics {
+    /// Last timestamp reported
+    last_reported: Instant,
+    /// Number of transactions scheduled
+    num_transactions_scheduled: usize,
+    /// Maximum pending_transactions length
+    max_pending_transactions: usize,
+    /// Maximum number of blocked transactions
+    max_blocked_transactions: usize,
+    /// Maximum executing transactions
+    max_executing_transactions: usize,
+
+    /// Total time spent processing completed transactions in microseconds
+    completed_transactions_time_us: u64,
+    /// Completed transactions - TryRecv time
+    compeleted_batch_try_recv_time_us: u64,
+    /// Completed transactions - Handle completed batch
+    completed_batch_handle_batch_time_us: u64,
+    completed_transactions_remove_account_locks_us: u64,
+    completed_transactions_unblock_transactions_us: u64,
+    /// Total time spent scheduling transactions in microseconds
+    scheduling_time_us: u64,
+    scheduling_try_schedule_time_us: u64,
+    check_blocking_transactions_time_us: u64,
+    get_conflicting_batches_time: u64,
+    find_thread_index_time: u64,
+    batching_time: u64,
+    max_batch_age: u64,
+    max_thread_index_used: usize,
+
+    /// Time spent checking and reporting metrics
+    metrics_us: u64,
+}
+
+impl Default for SchedulerMetrics {
+    fn default() -> Self {
+        Self {
+            last_reported: Instant::now(),
+            num_transactions_scheduled: Default::default(),
+            max_pending_transactions: Default::default(),
+            max_blocked_transactions: Default::default(),
+            max_executing_transactions: Default::default(),
+            completed_transactions_time_us: Default::default(),
+            scheduling_time_us: Default::default(),
+            compeleted_batch_try_recv_time_us: Default::default(),
+            completed_batch_handle_batch_time_us: Default::default(),
+            metrics_us: Default::default(),
+            completed_transactions_remove_account_locks_us: Default::default(),
+            completed_transactions_unblock_transactions_us: Default::default(),
+            scheduling_try_schedule_time_us: Default::default(),
+            check_blocking_transactions_time_us: Default::default(),
+            get_conflicting_batches_time: Default::default(),
+            find_thread_index_time: Default::default(),
+            batching_time: Default::default(),
+            max_batch_age: Default::default(),
+            max_thread_index_used: Default::default(),
+        }
+    }
+}
+
+impl SchedulerMetrics {
+    /// Report metrics if the interval has passed and reset metrics
+    fn report(&mut self) {
+        const REPORT_INTERVAL: std::time::Duration = std::time::Duration::from_millis(1000);
+
+        if self.last_reported.elapsed() >= REPORT_INTERVAL {
+            datapoint_info!(
+                "transaction-scheduler",
+                (
+                    "num_transactions_scheduled",
+                    self.num_transactions_scheduled as i64,
+                    i64
+                ),
+                (
+                    "max_pending_transactions",
+                    self.max_pending_transactions as i64,
+                    i64
+                ),
+                (
+                    "max_blocked_transactions",
+                    self.max_blocked_transactions as i64,
+                    i64
+                ),
+                (
+                    "max_executing_transactions",
+                    self.max_executing_transactions as i64,
+                    i64
+                ),
+                (
+                    "completed_transactions_time_us",
+                    self.completed_transactions_time_us as i64,
+                    i64
+                ),
+                (
+                    "compeleted_batch_try_recv_time_us",
+                    self.compeleted_batch_try_recv_time_us as i64,
+                    i64
+                ),
+                (
+                    "completed_batch_handle_batch_time_us",
+                    self.completed_batch_handle_batch_time_us as i64,
+                    i64
+                ),
+                (
+                    "completed_transactions_remove_account_locks_us",
+                    self.completed_transactions_remove_account_locks_us as i64,
+                    i64
+                ),
+                (
+                    "completed_transactions_unblock_transactions_us",
+                    self.completed_transactions_unblock_transactions_us as i64,
+                    i64
+                ),
+                ("scheduling_time_us", self.scheduling_time_us as i64, i64),
+                (
+                    "scheduling_try_schedule_time_us",
+                    self.scheduling_try_schedule_time_us as i64,
+                    i64
+                ),
+                (
+                    "check_blocking_transactions_time_us",
+                    self.check_blocking_transactions_time_us as i64,
+                    i64
+                ),
+                (
+                    "get_conflicting_batches_time",
+                    self.get_conflicting_batches_time as i64,
+                    i64
+                ),
+                (
+                    "find_thread_index_time",
+                    self.find_thread_index_time as i64,
+                    i64
+                ),
+                ("batching_time", self.batching_time as i64, i64),
+                ("metrics_us", self.metrics_us as i64, i64),
+                ("max_batch_age", self.max_batch_age as i64, i64),
+                (
+                    "max_thread_index_used",
+                    self.max_thread_index_used as i64,
+                    i64
+                ),
+            );
+
+            *self = Self::default();
+        }
+    }
+}

From a295b8e12f2e9f064231d3b0d54041cca6274d48 Mon Sep 17 00:00:00 2001
From: Andrew Fitzgerald <apfitzge@gmail.com>
Date: Tue, 2 Aug 2022 13:33:47 -0500
Subject: [PATCH 0003/3199] hook transaction scheduler into the bench

---
 transaction-scheduler-bench/src/main.rs | 21 ++++-----------------
 1 file changed, 4 insertions(+), 17 deletions(-)

diff --git a/transaction-scheduler-bench/src/main.rs b/transaction-scheduler-bench/src/main.rs
index a795dc5ed5acae..acfb1d172ef434 100644
--- a/transaction-scheduler-bench/src/main.rs
+++ b/transaction-scheduler-bench/src/main.rs
@@ -3,7 +3,10 @@ use {
     crossbeam_channel::{select, Receiver, Sender},
     log::info,
     rand::Rng,
-    solana_core::transaction_priority_details::GetTransactionPriorityDetails,
+    solana_core::{
+        transaction_priority_details::GetTransactionPriorityDetails,
+        transaction_scheduler::TransactionScheduler,
+    },
     solana_measure::measure,
     solana_perf::packet::{Packet, PacketBatch},
     solana_runtime::{bank::Bank, bank_forks::BankForks},
@@ -79,22 +82,6 @@ type TransactionMessage = Box<SanitizedTransaction>;
 type CompletedTransactionMessage = (usize, TransactionMessage); // thread index and transaction message
 type TransactionBatchMessage = Vec<TransactionMessage>;
 
-/// Dummy scheduler that you should replace with your own implementation
-struct TransactionScheduler;
-
-impl TransactionScheduler {
-    pub fn spawn_scheduler(
-        _packet_batch_receiver: Receiver<Vec<PacketBatch>>,
-        _transaction_batch_senders: Vec<Sender<TransactionBatchMessage>>,
-        _completed_transaction_receiver: Receiver<CompletedTransactionMessage>,
-        _bank_forks: Arc<RwLock<BankForks>>,
-        _max_batch_size: usize,
-        _exit: Arc<AtomicBool>,
-    ) -> JoinHandle<()> {
-        todo!()
-    }
-}
-
 #[derive(Debug, Default)]
 struct TransactionSchedulerBenchMetrics {
     /// Number of transactions sent to the scheduler

From f55102bad5fdfeced553c277d7caf65e68bce25f Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 23 Jul 2022 04:12:57 +0900
Subject: [PATCH 0004/3199] save

---
 Cargo.lock                 |  31 ++
 Cargo.toml                 |   1 +
 ledger-tool/Cargo.toml     |   2 +
 ledger-tool/src/main.rs    | 173 +++++++++-
 runtime/src/vote_parser.rs |   2 +-
 scheduler/Cargo.toml       |  30 ++
 scheduler/src/lib.rs       | 635 +++++++++++++++++++++++++++++++++++++
 scheduler/src/main.rs      | 104 ++++++
 8 files changed, 975 insertions(+), 3 deletions(-)
 create mode 100644 scheduler/Cargo.toml
 create mode 100644 scheduler/src/lib.rs
 create mode 100644 scheduler/src/main.rs

diff --git a/Cargo.lock b/Cargo.lock
index 14808c6088545d..ec5641cbf62a1b 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -248,6 +248,17 @@ dependencies = [
  "syn 1.0.98",
 ]
 
+[[package]]
+name = "atomic_enum"
+version = "0.1.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6a57588a20357c74ee9036656b7fad5c54328ee8657c6ff29ba31fd5ea503a1c"
+dependencies = [
+ "proc-macro2 1.0.38",
+ "quote 1.0.18",
+ "syn 1.0.93",
+]
+
 [[package]]
 name = "atty"
 version = "0.2.14"
@@ -5512,8 +5523,10 @@ dependencies = [
  "solana-ledger",
  "solana-logger 1.11.5",
  "solana-measure",
+ "solana-metrics",
  "solana-runtime",
  "solana-sdk 1.11.5",
+ "solana-scheduler",
  "solana-stake-program",
  "solana-storage-bigtable",
  "solana-transaction-status",
@@ -6043,6 +6056,24 @@ dependencies = [
  "zstd",
 ]
 
+[[package]]
+name = "solana-scheduler"
+version = "1.11.4"
+dependencies = [
+ "atomic_enum",
+ "crossbeam-channel",
+ "log",
+ "rand 0.7.3",
+ "sha2 0.10.2",
+ "solana-entry",
+ "solana-logger 1.11.4",
+ "solana-measure",
+ "solana-metrics",
+ "solana-runtime",
+ "solana-sdk 1.11.4",
+ "solana-version",
+]
+
 [[package]]
 name = "solana-sdk"
 version = "1.10.33"
diff --git a/Cargo.toml b/Cargo.toml
index ef847c1024a17d..9e7ee460819c51 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -65,6 +65,7 @@ members = [
     "rpc-test",
     "runtime",
     "runtime/store-tool",
+    "scheduler",
     "sdk",
     "sdk/cargo-build-bpf",
     "sdk/cargo-build-sbf",
diff --git a/ledger-tool/Cargo.toml b/ledger-tool/Cargo.toml
index 740d4fc7322b6a..ffc3dc8dda0517 100644
--- a/ledger-tool/Cargo.toml
+++ b/ledger-tool/Cargo.toml
@@ -28,6 +28,7 @@ solana-core = { path = "../core", version = "=1.11.5" }
 solana-entry = { path = "../entry", version = "=1.11.5" }
 solana-ledger = { path = "../ledger", version = "=1.11.5" }
 solana-logger = { path = "../logger", version = "=1.11.5" }
+solana-metrics = { path = "../metrics", version = "=1.11.5" }
 solana-measure = { path = "../measure", version = "=1.11.5" }
 solana-runtime = { path = "../runtime", version = "=1.11.5" }
 solana-sdk = { path = "../sdk", version = "=1.11.5" }
@@ -35,6 +36,7 @@ solana-stake-program = { path = "../programs/stake", version = "=1.11.5" }
 solana-storage-bigtable = { path = "../storage-bigtable", version = "=1.11.5" }
 solana-transaction-status = { path = "../transaction-status", version = "=1.11.5" }
 solana-version = { path = "../version", version = "=1.11.5" }
+solana-scheduler = { path = "../scheduler", version = "=1.11.5" }
 solana-vote-program = { path = "../programs/vote", version = "=1.11.5" }
 tokio = { version = "~1.14.1", features = ["full"] }
 
diff --git a/ledger-tool/src/main.rs b/ledger-tool/src/main.rs
index a91e9888d6871c..bc34c0139a214f 100644
--- a/ledger-tool/src/main.rs
+++ b/ledger-tool/src/main.rs
@@ -52,6 +52,7 @@ use {
             DEFAULT_MAX_INCREMENTAL_SNAPSHOT_ARCHIVES_TO_RETAIN, SUPPORTED_ARCHIVE_COMPRESSION,
         },
     },
+    solana_scheduler::{AddressBook, ScheduleStage, TaskQueue, Weight, LockAttempt, RequestedUsage},
     solana_sdk::{
         account::{AccountSharedData, ReadableAccount, WritableAccount},
         account_utils::StateMut,
@@ -140,9 +141,12 @@ fn output_entry(
     slot: Slot,
     entry_index: usize,
     entry: Entry,
+    to_schedule_stage: &mut Vec<Box<(SanitizedTransaction, Vec<LockAttempt>)>>,
+    skip_voting: bool,
 ) {
     match method {
         LedgerOutputMethod::Print => {
+            /*
             println!(
                 "  Entry {} - num_hashes: {}, hash: {}, transactions: {}",
                 entry_index,
@@ -150,8 +154,33 @@ fn output_entry(
                 entry.hash,
                 entry.transactions.len()
             );
+            */
             for (transactions_index, transaction) in entry.transactions.into_iter().enumerate() {
-                println!("    Transaction {}", transactions_index);
+                //println!("    Transaction {}", transactions_index);
+                let sanitized_tx = SanitizedTransaction::try_create(
+                    transaction.clone(),
+                    MessageHash::Compute,
+                    None,
+                    SimpleAddressLoader::Disabled,
+                    true, // require_static_program_ids
+                )
+                .unwrap();
+                if !skip_voting
+                    || !solana_runtime::vote_parser::is_simple_vote_transaction(&sanitized_tx)
+                {
+                    let locks = sanitized_tx.get_account_locks().unwrap();
+                    let writable_lock_iter = locks
+                        .writable
+                        .iter()
+                        .map(|address| LockAttempt::new(**address, RequestedUsage::Writable));
+                    let readonly_lock_iter = locks
+                        .readonly
+                        .iter()
+                        .map(|address| LockAttempt::new(**address, RequestedUsage::Readonly));
+                    let locks = writable_lock_iter.chain(readonly_lock_iter).collect::<Vec<_>>();
+                    to_schedule_stage.push(Box::new((sanitized_tx, locks)));
+                }
+                /*
                 let tx_signature = transaction.signatures[0];
                 let tx_status_meta = blockstore
                     .read_transaction_status((tx_signature, slot))
@@ -171,6 +200,7 @@ fn output_entry(
                     None,
                     None,
                 );
+                */
             }
         }
         LedgerOutputMethod::Json => {
@@ -219,10 +249,149 @@ fn output_slot(
         }
     }
 
+    let (muxed_sender, muxed_receiver) = crossbeam_channel::unbounded();
+
+    // this should be target number of saturated cpu cores
+    let lane_count = std::env::var("EXECUTION_LANE_COUNT")
+        .unwrap_or(format!("{}", std::thread::available_parallelism().unwrap()))
+        .parse::<usize>()
+        .unwrap();
+    let lane_channel_factor = std::env::var("LANE_CHANNEL_FACTOR")
+        .unwrap_or(format!("{}", std::thread::available_parallelism().unwrap()))
+        .parse::<usize>()
+        .unwrap();
+    //let (pre_execute_env_sender, pre_execute_env_receiver) = crossbeam_channel::bounded(lane_count * lane_channel_factor);
+    let (pre_execute_env_sender, pre_execute_env_receiver) = crossbeam_channel::unbounded();
+
+    //let (pre_execute_env_sender, pre_execute_env_receiver) = crossbeam_channel::unbounded();
+    //let (post_execute_env_sender, post_execute_env_receiver) = crossbeam_channel::unbounded();
+    //
+    let (post_schedule_env_sender, post_schedule_env_receiver) = crossbeam_channel::unbounded();
+    let mut runnable_queue = TaskQueue::default();
+    let mut contended_queue = TaskQueue::default();
+    let mut address_book = AddressBook::default();
+    let t1 = std::thread::Builder::new()
+        .name("sol-scheduler".to_string())
+        .spawn(move || loop {
+            ScheduleStage::run(
+                lane_count * lane_channel_factor,
+                &mut runnable_queue,
+                &mut contended_queue,
+                &mut address_book,
+                &muxed_receiver,
+                &pre_execute_env_sender,
+                &post_schedule_env_sender,
+            );
+        })
+        .unwrap();
+    let handles = (0..lane_count)
+        .map(|thx| {
+            let pre_execute_env_receiver = pre_execute_env_receiver.clone();
+            let muxed_sender = muxed_sender.clone();
+
+            let t2 = std::thread::Builder::new()
+                .name(format!("blockstore_processor_{}", thx))
+                .spawn(move || {
+                    use solana_metrics::datapoint_info;
+                    let current_thread_name = std::thread::current().name().unwrap().to_string();
+                    let send_metrics = std::env::var("SEND_METRICS").is_ok();
+
+                    for step in 0.. {
+                        let ee = pre_execute_env_receiver.recv().unwrap();
+                        if step % 1966 == 0 {
+                            error!("executing!: {} {}", step, pre_execute_env_receiver.len());
+                        }
+
+                        if send_metrics {
+                            let mut process_message_time = Measure::start("process_message_time");
+                            let sig = ee.task.tx.0.signature().to_string();
+                            trace!("execute substage: #{} {:#?}", step, &sig);
+                            std::thread::sleep(std::time::Duration::from_micros(
+                                ee.cu.try_into().unwrap(),
+                            ));
+
+                            process_message_time.stop();
+                            let duration_with_overhead = process_message_time.as_us();
+
+                            datapoint_info!(
+                                "individual_tx_stats",
+                                ("slot", 33333, i64),
+                                ("thread", current_thread_name, String),
+                                ("signature", &sig, String),
+                                ("account_locks_in_json", "{}", String),
+                                ("status", "Ok", String),
+                                ("duration", duration_with_overhead, i64),
+                                ("compute_units", ee.cu, i64),
+                            );
+                        }
+
+                        muxed_sender
+                            .send(solana_scheduler::MultiplexedPayload::FromExecute(ee))
+                            .unwrap();
+                    }
+                })
+                .unwrap();
+            t2
+        })
+        .collect::<Vec<_>>();
+
+    let depth = Arc::new(std::sync::atomic::AtomicUsize::default());
+
+    let d = depth.clone();
+    let t3 = std::thread::Builder::new()
+        .name("sol-consumer".to_string())
+        .spawn(move || {
+            for step in 0.. {
+                let ee = post_schedule_env_receiver.recv().unwrap();
+                d.fetch_sub(1, Ordering::Relaxed);
+                trace!(
+                    "post schedule stage: #{} {:#?}",
+                    step,
+                    ee.task.tx.0.signature()
+                );
+                if step % 1966 == 0 {
+                    error!("finished!: {} {}", step, post_schedule_env_receiver.len());
+                }
+            }
+        })
+        .unwrap();
+
     if verbose_level >= 2 {
+        let mut txes = Vec::new();
+        let skip_voting = std::env::var("SKIP_VOTING").is_ok();
         for (entry_index, entry) in entries.into_iter().enumerate() {
-            output_entry(blockstore, method, slot, entry_index, entry);
+            output_entry(
+                blockstore,
+                method,
+                slot,
+                entry_index,
+                entry,
+                &mut txes,
+                skip_voting,
+            );
+        }
+
+        let mut weight = 10_000_000;
+        for i in 0..10000 {
+            error!("started!: {} {}", i, txes.len());
+            for tx in txes.clone() {
+                while depth.load(Ordering::Relaxed) > 10_000 {
+                    std::thread::sleep(std::time::Duration::from_micros(10));
+                }
+
+                muxed_sender
+                    .send(solana_scheduler::MultiplexedPayload::FromPrevious((
+                        Weight { ix: weight },
+                        tx,
+                    )))
+                    .unwrap();
+                depth.fetch_add(1, Ordering::Relaxed);
+                weight -= 1;
+            }
         }
+        t1.join().unwrap();
+        handles.into_iter().for_each(|t| t.join().unwrap());
+        t3.join().unwrap();
 
         output_slot_rewards(blockstore, slot, method);
     } else if verbose_level >= 1 {
diff --git a/runtime/src/vote_parser.rs b/runtime/src/vote_parser.rs
index c67045b0827533..e761772982b6c7 100644
--- a/runtime/src/vote_parser.rs
+++ b/runtime/src/vote_parser.rs
@@ -13,7 +13,7 @@ use {
 pub type ParsedVote = (Pubkey, VoteTransaction, Option<Hash>, Signature);
 
 // Used for filtering out votes from the transaction log collector
-pub(crate) fn is_simple_vote_transaction(transaction: &SanitizedTransaction) -> bool {
+pub fn is_simple_vote_transaction(transaction: &SanitizedTransaction) -> bool {
     if transaction.message().instructions().len() == 1 {
         let (program_pubkey, instruction) = transaction
             .message()
diff --git a/scheduler/Cargo.toml b/scheduler/Cargo.toml
new file mode 100644
index 00000000000000..e617b893059139
--- /dev/null
+++ b/scheduler/Cargo.toml
@@ -0,0 +1,30 @@
+[package]
+authors = ["Solana Maintainers <maintainers@solana.foundation>"]
+edition = "2021"
+name = "solana-scheduler"
+description = "The solana scheduler"
+version = "1.11.4"
+repository = "https://github.com/solana-labs/solana"
+license = "Apache-2.0"
+homepage = "https://solana.com/"
+documentation = "https://docs.rs/solana-scheduler"
+publish = true
+
+[dependencies]
+atomic_enum = "0.1.1"
+crossbeam-channel = "0.5.5"
+log = "0.4.17"
+solana-entry = { path = "../entry", version = "=1.11.4" }
+solana-runtime = { path = "../runtime", version = "=1.11.4" }
+solana-sdk = { path = "../sdk", version = "=1.11.4" }
+solana-logger = { path = "../logger", version = "=1.11.4" }
+solana-version = { path = "../version", version = "=1.11.4" }
+solana-metrics = { path = "../metrics", version = "=1.11.4" }
+solana-measure = { path = "../measure", version = "=1.11.4" }
+sha2 = { version = "0.10.2" }
+rand = "0.7.0"
+
+
+[[bin]]
+name = "solana-scheduler"
+path = "src/main.rs"
diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
new file mode 100644
index 00000000000000..b3cca1d2231d50
--- /dev/null
+++ b/scheduler/src/lib.rs
@@ -0,0 +1,635 @@
+#![feature(map_first_last)]
+
+use {
+    atomic_enum::atomic_enum,
+    crossbeam_channel::{bounded, unbounded},
+    log::*,
+    rand::Rng,
+    sha2::{Digest, Sha256},
+    solana_entry::entry::Entry,
+    solana_measure::measure::Measure,
+    solana_metrics::datapoint_info,
+    solana_sdk::{
+        hash::Hash,
+        pubkey::Pubkey,
+        transaction::{SanitizedTransaction, TransactionAccountLocks, VersionedTransaction},
+    },
+};
+
+#[derive(Debug)]
+pub struct ExecutionEnvironment {
+    lock_attempts: Vec<LockAttempt>,
+    //accounts: Vec<i8>,
+    pub cu: usize,
+    pub task: Task,
+}
+
+impl ExecutionEnvironment {
+    //fn new(cu: usize) -> Self {
+    //    Self {
+    //        cu,
+    //        ..Self::default()
+    //    }
+    //}
+
+    //fn abort() {
+    //  pass AtomicBool into InvokeContext??
+    //}
+}
+
+#[derive(Clone, Debug)]
+pub struct LockAttempt {
+    address: Pubkey,
+    is_success: bool,
+    requested_usage: RequestedUsage,
+}
+
+impl LockAttempt {
+    fn is_success(&self) -> bool {
+        self.is_success
+    }
+
+    fn is_failed(&self) -> bool {
+        !self.is_success()
+    }
+
+    pub fn new(address: Pubkey, requested_usage: RequestedUsage) -> Self {
+        Self {
+            address,
+            is_success: true,
+            requested_usage,
+        }
+    }
+}
+
+type UsageCount = usize;
+const SOLE_USE_COUNT: UsageCount = 1;
+
+#[derive(PartialEq)]
+enum CurrentUsage {
+    Unused,
+    // weight to abort running tx?
+    // also sum all readonly weights to subvert to write lock with greater weight?
+    Readonly(UsageCount),
+    Writable,
+}
+
+impl CurrentUsage {
+    fn renew(requested_usage: RequestedUsage) -> Self {
+        match requested_usage {
+            RequestedUsage::Readonly => CurrentUsage::Readonly(SOLE_USE_COUNT),
+            RequestedUsage::Writable => CurrentUsage::Writable,
+        }
+    }
+}
+
+#[derive(Clone, Copy, Debug)]
+pub enum RequestedUsage {
+    Readonly,
+    Writable,
+}
+
+struct Page {
+    current_usage: CurrentUsage,
+    contended_unique_weights: std::collections::BTreeSet<UniqueWeight>,
+    //next_scheduled_task // reserved_task guaranteed_task
+    //loaded account from Accounts db
+    //comulative_cu for qos; i.e. track serialized cumulative keyed by addresses and bail out block
+    //producing as soon as any one of cu from the executing thread reaches to the limit
+}
+
+impl Page {
+    fn new(current_usage: CurrentUsage) -> Self {
+        Self {
+            current_usage,
+            contended_unique_weights: Default::default(),
+        }
+    }
+}
+
+type AddressMap = std::collections::HashMap<Pubkey, Page>;
+type AddressSet = std::collections::HashSet<Pubkey>;
+type AddressMapEntry<'a, K, V> = std::collections::hash_map::Entry<'a, K, V>;
+
+// needs ttl mechanism and prune
+#[derive(Default)]
+pub struct AddressBook {
+    book: AddressMap,
+    newly_uncontended_addresses: AddressSet,
+}
+
+impl AddressBook {
+    #[inline(never)]
+    fn attempt_lock_address(
+        &mut self,
+        from_runnable: bool,
+        unique_weight: &UniqueWeight,
+        attempt: &mut LockAttempt,
+    ) {
+        let LockAttempt {address, requested_usage, is_success} = attempt;
+
+        match self.book.entry(*address) {
+            AddressMapEntry::Vacant(book_entry) => {
+                book_entry.insert(Page::new(CurrentUsage::renew(*requested_usage)));
+                *is_success = true;
+            }
+            AddressMapEntry::Occupied(mut book_entry) => {
+                let mut page = book_entry.get_mut();
+
+                match &mut page.current_usage {
+                    CurrentUsage::Unused => {
+                        page.current_usage = CurrentUsage::renew(*requested_usage);
+                        *is_success = true;
+                    }
+                    CurrentUsage::Readonly(ref mut count) => match requested_usage {
+                        RequestedUsage::Readonly => {
+                            *count += 1;
+                            *is_success = true;
+                        }
+                        RequestedUsage::Writable => {
+                            if from_runnable {
+                                Self::remember_address_contention(page, unique_weight);
+                            }
+                            *is_success = false;
+                        }
+                    },
+                    CurrentUsage::Writable => match requested_usage {
+                        RequestedUsage::Readonly | RequestedUsage::Writable => {
+                            if from_runnable {
+                                Self::remember_address_contention(page, unique_weight);
+                            }
+                            *is_success = false;
+                        }
+                    },
+                }
+            }
+        }
+    }
+
+    #[inline(never)]
+    fn remember_address_contention(page: &mut Page, unique_weight: &UniqueWeight) {
+        page.contended_unique_weights.insert(*unique_weight);
+    }
+
+    #[inline(never)]
+    fn forget_address_contention(&mut self, unique_weight: &UniqueWeight, address: &Pubkey) {
+        match self.book.entry(*address) {
+            AddressMapEntry::Vacant(_book_entry) => unreachable!(),
+            AddressMapEntry::Occupied(mut entry) => {
+                let page = entry.get_mut();
+                page.contended_unique_weights.remove(unique_weight);
+            }
+        }
+    }
+
+    fn ensure_unlock(&mut self, attempt: &LockAttempt) {
+        if attempt.is_success() {
+            self.unlock(attempt);
+        }
+    }
+
+    #[inline(never)]
+    fn unlock(&mut self, attempt: &LockAttempt) -> bool {
+        debug_assert!(attempt.is_success());
+
+        let mut newly_uncontended = false;
+        let mut still_queued = false;
+
+        match self.book.entry(attempt.address) {
+            AddressMapEntry::Occupied(mut book_entry) => {
+                let mut page = book_entry.get_mut();
+
+                match &mut page.current_usage {
+                    CurrentUsage::Readonly(ref mut count) => match &attempt.requested_usage {
+                        RequestedUsage::Readonly => {
+                            if *count == SOLE_USE_COUNT {
+                                newly_uncontended = true;
+                            } else {
+                                *count -= 1;
+                            }
+                        }
+                        RequestedUsage::Writable => unreachable!(),
+                    },
+                    CurrentUsage::Writable => match &attempt.requested_usage {
+                        RequestedUsage::Writable => {
+                            newly_uncontended = true;
+                        }
+                        RequestedUsage::Readonly => unreachable!(),
+                    },
+                    CurrentUsage::Unused => unreachable!(),
+                }
+
+                if newly_uncontended {
+                    page.current_usage = CurrentUsage::Unused;
+                    if !page.contended_unique_weights.is_empty() {
+                        still_queued = true;
+                    }
+                }
+            }
+            AddressMapEntry::Vacant(_book_entry) => {
+                unreachable!()
+            }
+        }
+
+        still_queued
+    }
+}
+
+#[derive(Copy, Clone, Debug, PartialEq, Eq, PartialOrd, Ord)]
+pub struct Weight {
+    // naming: Sequence Ordering?
+    pub ix: usize, // index in ledger entry?
+                   // gas fee
+}
+
+#[derive(Copy, Clone, Debug, PartialEq, Eq, PartialOrd, Ord)]
+pub struct UniqueWeight {
+    // naming: Sequence Ordering?
+    weight: Weight,
+    // we can't use Transaction::message_hash because it's manipulatable to be favorous to the tx
+    // submitter
+    //unique_key: Hash, // tie breaker? random noise? also for unique identification of txes?
+    // fee?
+}
+/*
+pub type Weight = usize;
+pub type UniqueWeight = usize;
+*/
+
+struct Bundle {
+    // what about bundle1{tx1a, tx2} and bundle2{tx1b, tx2}?
+}
+
+#[derive(Debug)]
+pub struct Task {
+    pub tx: Box<(SanitizedTransaction, Vec<LockAttempt>)>, // actually should be Bundle
+}
+
+// RunnableQueue, ContendedQueue?
+#[derive(Default)]
+pub struct TaskQueue {
+    tasks: std::collections::BTreeMap<UniqueWeight, Task>,
+}
+
+impl TaskQueue {
+    #[inline(never)]
+    fn add_to_schedule(&mut self, unique_weight: UniqueWeight, task: Task) {
+        //trace!("TaskQueue::add(): {:?}", unique_weight);
+        let pre_existed = self.tasks.insert(unique_weight, task);
+        debug_assert!(pre_existed.is_none()); //, "identical shouldn't exist: {:?}", unique_weight);
+    }
+
+    #[inline(never)]
+    fn entry_to_execute(
+        &mut self,
+        unique_weight: UniqueWeight,
+    ) -> std::collections::btree_map::OccupiedEntry<'_, UniqueWeight, Task> {
+        use std::collections::btree_map::Entry;
+
+        let queue_entry = self.tasks.entry(unique_weight);
+        match queue_entry {
+            Entry::Occupied(queue_entry) => queue_entry,
+            Entry::Vacant(_queue_entry) => unreachable!(),
+        }
+    }
+
+    #[inline(never)]
+    fn heaviest_entry_to_execute(
+        &mut self,
+    ) -> Option<std::collections::btree_map::OccupiedEntry<'_, UniqueWeight, Task>> {
+        self.tasks.last_entry()
+    }
+}
+
+#[inline(never)]
+fn attempt_lock_for_execution<'a>(
+    from_runnable: bool,
+    address_book: &mut AddressBook,
+    unique_weight: &UniqueWeight,
+    message_hash: &'a Hash,
+    mut placeholder_attempts: Vec<LockAttempt>,
+) -> (bool, Vec<LockAttempt>) {
+    // no short-cuircuit; we at least all need to add to the contended queue
+    let mut all_succeeded_so_far = true;
+
+    for attempt in placeholder_attempts.iter_mut() {
+        address_book.attempt_lock_address(from_runnable, unique_weight, attempt);
+        if all_succeeded_so_far && attempt.is_failed() {
+            all_succeeded_so_far = false;
+        }
+    }
+
+    (all_succeeded_so_far, placeholder_attempts)
+}
+
+// multiplexed to reduce the futex syscal per tx down to minimum and to make the schduler to
+// adaptive relative load between sigverify stage and execution substage
+// switched from crossbeam_channel::select! due to observed poor performance
+pub enum MultiplexedPayload {
+    FromPrevious((Weight, Box<(SanitizedTransaction, Vec<LockAttempt>)>)),
+    FromExecute(Box<ExecutionEnvironment>),
+}
+
+pub struct ScheduleStage {}
+
+impl ScheduleStage {
+    fn push_to_queue(
+        (weight, tx): (Weight, Box<(SanitizedTransaction, Vec<LockAttempt>)>),
+        runnable_queue: &mut TaskQueue,
+    ) {
+        // manage randomness properly for future scheduling determinism
+        //let mut rng = rand::thread_rng();
+
+        //let ix = 23;
+        //let tx = bank
+        //    .verify_transaction(
+        //        tx,
+        //        solana_sdk::transaction::TransactionVerificationMode::FullVerification,
+        //    )
+        //    .unwrap();
+        //tx.foo();
+
+        runnable_queue.add_to_schedule(
+            UniqueWeight {
+                weight,
+                //unique_key: solana_sdk::hash::new_rand(&mut rng),
+            },
+            Task { tx },
+        );
+    }
+
+    #[inline(never)]
+    fn get_newly_u_u_w<'a>(
+        address: &'a Pubkey,
+        address_book: &'a AddressBook,
+    ) -> &'a std::collections::BTreeSet<UniqueWeight> {
+        &address_book
+            .book
+            .get(address)
+            .unwrap()
+            .contended_unique_weights
+    }
+
+    #[inline(never)]
+    fn get_weight_from_contended(address_book: &AddressBook) -> Option<UniqueWeight> {
+        let mut heaviest_weight: Option<UniqueWeight> = None;
+        //trace!("n u a len(): {}", address_book.newly_uncontended_addresses.len());
+        for address in address_book.newly_uncontended_addresses.iter() {
+            let newly_uncontended_unique_weights = Self::get_newly_u_u_w(address, &address_book);
+            if let Some(&weight) = newly_uncontended_unique_weights.last() {
+                if let Some(current_heaviest_weight) = heaviest_weight {
+                    if weight > current_heaviest_weight {
+                        heaviest_weight = Some(weight);
+                    }
+                } else {
+                    heaviest_weight = Some(weight);
+                }
+            }
+        }
+        heaviest_weight
+    }
+
+    #[inline(never)]
+    fn select_next_task<'a>(
+        runnable_queue: &'a mut TaskQueue,
+        contended_queue: &'a mut TaskQueue,
+        address_book: &mut AddressBook,
+    ) -> Option<(
+        Option<&'a mut TaskQueue>,
+        std::collections::btree_map::OccupiedEntry<'a, UniqueWeight, Task>,
+    )> {
+        match (
+            runnable_queue.heaviest_entry_to_execute(),
+            Self::get_weight_from_contended(address_book),
+        ) {
+            (Some(heaviest_runnable_entry), None) => {
+                Some((Some(contended_queue), heaviest_runnable_entry))
+            }
+            (None, Some(weight_from_contended)) => Some((
+                None,
+                contended_queue.entry_to_execute(weight_from_contended),
+            )),
+            (Some(heaviest_runnable_entry), Some(weight_from_contended)) => {
+                let weight_from_runnable = heaviest_runnable_entry.key();
+
+                if weight_from_runnable > &weight_from_contended {
+                    Some((Some(contended_queue), heaviest_runnable_entry))
+                } else if &weight_from_contended > weight_from_runnable {
+                    Some((
+                        None,
+                        contended_queue.entry_to_execute(weight_from_contended),
+                    ))
+                } else {
+                    unreachable!(
+                        "identical unique weights shouldn't exist in both runnable and contended"
+                    )
+                }
+            }
+            (None, None) => None,
+        }
+    }
+
+    #[inline(never)]
+    fn pop_from_queue_then_lock(
+        runnable_queue: &mut TaskQueue,
+        contended_queue: &mut TaskQueue,
+        address_book: &mut AddressBook,
+    ) -> Option<(UniqueWeight, Task, Vec<LockAttempt>)> {
+        for (reborrowed_contended_queue, mut queue_entry) in
+            Self::select_next_task(runnable_queue, contended_queue, address_book)
+        {
+            let from_runnable = reborrowed_contended_queue.is_some();
+            let unique_weight = *queue_entry.key();
+            let next_task = queue_entry.get_mut();
+            let message_hash = next_task.tx.0.message_hash();
+            let placeholder_lock_attempts = std::mem::take(&mut next_task.tx.1);
+
+            // plumb message_hash into StatusCache or implmenent our own for duplicate tx
+            // detection?
+
+            let (is_success, mut populated_lock_attempts) = attempt_lock_for_execution(
+                from_runnable,
+                address_book,
+                &unique_weight,
+                &message_hash,
+                placeholder_lock_attempts,
+            );
+
+            if !is_success {
+                //trace!("ensure_unlock_for_failed_execution(): {:?} {}", (&unique_weight, from_runnable), next_task.tx.signature());
+                Self::ensure_unlock_for_failed_execution(
+                    address_book,
+                    &populated_lock_attempts,
+                    from_runnable,
+                );
+                std::mem::swap(&mut next_task.tx.1, &mut populated_lock_attempts);
+                if from_runnable {
+                    reborrowed_contended_queue
+                        .unwrap()
+                        .add_to_schedule(*queue_entry.key(), queue_entry.remove());
+                    // maybe run lightweight prune logic on contended_queue here.
+                }
+                continue;
+            }
+
+            Self::finalize_successful_lock_before_execution(
+                address_book,
+                &unique_weight,
+                &populated_lock_attempts,
+            );
+            let task = queue_entry.remove();
+            return Some((unique_weight, task, populated_lock_attempts));
+        }
+
+        None
+    }
+
+    #[inline(never)]
+    // naming: relock_before_execution() / update_address_book() / update_uncontended_addresses()?
+    fn finalize_successful_lock_before_execution(
+        address_book: &mut AddressBook,
+        unique_weight: &UniqueWeight,
+        lock_attempts: &Vec<LockAttempt>,
+    ) {
+        for l in lock_attempts {
+            // ensure to remove remaining refs of this unique_weight
+            address_book.forget_address_contention(&unique_weight, &l.address);
+
+            // revert because now contended again
+            address_book.newly_uncontended_addresses.remove(&l.address);
+        }
+    }
+
+    #[inline(never)]
+    fn ensure_unlock_for_failed_execution(
+        address_book: &mut AddressBook,
+        lock_attempts: &Vec<LockAttempt>,
+        from_runnable: bool,
+    ) {
+        for l in lock_attempts {
+            address_book.ensure_unlock(&l);
+
+            // revert because now contended again
+            if !from_runnable {
+                address_book.newly_uncontended_addresses.remove(&l.address);
+            }
+
+            // todo: mem::forget and panic in LockAttempt::drop()
+        }
+    }
+
+    #[inline(never)]
+    fn unlock_after_execution(address_book: &mut AddressBook, lock_attempts: Vec<LockAttempt>) {
+        for l in lock_attempts {
+            let newly_uncontended_while_queued = address_book.unlock(&l);
+            if newly_uncontended_while_queued {
+                address_book.newly_uncontended_addresses.insert(l.address);
+            }
+
+            // todo: mem::forget and panic in LockAttempt::drop()
+        }
+    }
+
+    #[inline(never)]
+    fn prepare_scheduled_execution(
+        address_book: &mut AddressBook,
+        unique_weight: UniqueWeight,
+        task: Task,
+        lock_attempts: Vec<LockAttempt>,
+    ) -> Box<ExecutionEnvironment> {
+        let mut rng = rand::thread_rng();
+        // load account now from AccountsDb
+
+        Box::new(ExecutionEnvironment {
+            lock_attempts,
+            task,
+            cu: rng.gen_range(3, 1000),
+        })
+    }
+
+    #[inline(never)]
+    fn commit_result(ee: &mut ExecutionEnvironment, address_book: &mut AddressBook) {
+        let lock_attempts = std::mem::take(&mut ee.lock_attempts);
+        // do par()-ly?
+        Self::unlock_after_execution(address_book, lock_attempts);
+        // block-wide qos validation will be done here
+        // if error risen..:
+        //   don't commit the tx for banking and potentially finish scheduling at block max cu
+        //   limit
+        //   mark the block as dead for replaying
+
+        // par()-ly clone updated Accounts into address book
+    }
+
+    #[inline(never)]
+    fn schedule_next_execution(
+        runnable_queue: &mut TaskQueue,
+        contended_queue: &mut TaskQueue,
+        address_book: &mut AddressBook,
+    ) -> Option<Box<ExecutionEnvironment>> {
+        let maybe_ee =
+            Self::pop_from_queue_then_lock(runnable_queue, contended_queue, address_book)
+                .map(|(uw, t, ll)| Self::prepare_scheduled_execution(address_book, uw, t, ll));
+        maybe_ee
+    }
+
+    #[inline(never)]
+    fn register_runnable_task(
+        weighted_tx: (Weight, Box<(SanitizedTransaction, Vec<LockAttempt>)>),
+        runnable_queue: &mut TaskQueue,
+    ) {
+        Self::push_to_queue(weighted_tx, runnable_queue)
+    }
+
+    pub fn run(
+        max_executing_queue_count: usize,
+        runnable_queue: &mut TaskQueue,
+        contended_queue: &mut TaskQueue,
+        address_book: &mut AddressBook,
+        from: &crossbeam_channel::Receiver<MultiplexedPayload>,
+        to_execute_substage: &crossbeam_channel::Sender<Box<ExecutionEnvironment>>,
+        to_next_stage: &crossbeam_channel::Sender<Box<ExecutionEnvironment>>, // assume nonblocking
+    ) {
+        let mut executing_queue_count = 0;
+
+        loop {
+            trace!("schedule_once!");
+
+            let i = from.recv().unwrap();
+            match i {
+                MultiplexedPayload::FromPrevious(weighted_tx) => {
+                    trace!("recv from previous");
+
+                    Self::register_runnable_task(weighted_tx, runnable_queue);
+                }
+                MultiplexedPayload::FromExecute(mut processed_execution_environment) => {
+                    trace!("recv from execute");
+                    executing_queue_count -= 1;
+
+                    Self::commit_result(&mut processed_execution_environment, address_book);
+                    // async-ly propagate the result to rpc subsystems
+                    to_next_stage.send(processed_execution_environment).unwrap();
+                }
+            }
+
+            while executing_queue_count < max_executing_queue_count {
+                let maybe_ee =
+                    Self::schedule_next_execution(runnable_queue, contended_queue, address_book);
+                if let Some(ee) = maybe_ee {
+                    trace!("send to execute");
+                    executing_queue_count += 1;
+
+                    to_execute_substage.send(ee).unwrap();
+                } else {
+                    break;
+                }
+            }
+        }
+    }
+}
+
+struct ExecuteStage {
+    //bank: Bank,
+}
+
+impl ExecuteStage {}
diff --git a/scheduler/src/main.rs b/scheduler/src/main.rs
new file mode 100644
index 00000000000000..233195354c5409
--- /dev/null
+++ b/scheduler/src/main.rs
@@ -0,0 +1,104 @@
+fn main() {
+    solana_logger::setup();
+    error!("hello");
+    let thread_count = 10;
+    let (s, r) = bounded::<(
+        (usize, usize, (std::time::Instant, ExecutionEnvironment)),
+        Vec<u8>,
+    )>(thread_count * 10);
+    let (s2, r2) = bounded(thread_count * 2);
+
+    /*
+    let p = std::thread::Builder::new().name("producer".to_string()).spawn(move || {
+        let mut rng = rand::thread_rng();
+        loop {
+            s2.send((std::time::Instant::now(), ExecutionEnvironment::new(rng.gen_range(0, 1000)))).unwrap();
+        }
+    }).unwrap();
+    */
+    let pc = std::thread::Builder::new().name("prosumer".to_string()).spawn(move || {
+        use crossbeam_channel::select;
+
+        let mut rng = rand::thread_rng();
+        let mut count = 0;
+        let start = std::time::Instant::now();
+        //let mut rrr = Vec::with_capacity(10);
+        //for _ in 0..100 {
+        let mut elapsed = 0;
+
+        loop {
+            select! {
+                send(s2, (std::time::Instant::now(), ExecutionEnvironment::new(rng.gen_range(0, 1000)))) -> res => {
+                    res.unwrap();
+                }
+                recv(r) -> msg => {
+                    let rr = msg.unwrap();
+                    elapsed += rr.0.2.0.elapsed().as_nanos();
+                    //    rrr.push((rr.0.2.0.elapsed(), rr));
+                    //}
+
+                    //for rr in rrr {
+                    count += 1;
+                    //error!("recv-ed: {:?}", &rr);
+                    if count % 100_000 == 0 {
+                        error!("recv-ed: {}", count / start.elapsed().as_secs().max(1));
+                        //break
+                    }
+                }
+                }
+        }
+    }).unwrap();
+
+    let mut joins = (0..thread_count)
+        .map(|thx| {
+            let s = s.clone();
+            let r2 = r2.clone();
+            std::thread::Builder::new()
+                .name(format!("blockstore_processor_{}", thx))
+                .spawn(move || {
+                    let current_thread_name = std::thread::current().name().unwrap().to_string();
+                    let mut i = 0;
+                    //for _ in 0..60 {//000000 {
+                    loop {
+                        let ss = (thx, i, r2.recv().unwrap());
+
+                        let mut process_message_time = Measure::start("process_message_time");
+
+                        let mut hasher = Sha256::default();
+                        let cu = ss.2 .1.cu;
+                        for i in 0_usize..cu {
+                            //for _ in 0..10 {
+                            hasher.update(i.to_le_bytes());
+                            //}
+                        }
+                        let h = hasher.finalize();
+
+                        process_message_time.stop();
+                        let duration_with_overhead = process_message_time.as_us();
+
+                        /*
+                        datapoint_info!(
+                            "individual_tx_stats",
+                            ("slot", 33333, i64),
+                            ("thread", current_thread_name, String),
+                            ("signature", "ffffff", String),
+                            ("account_locks_in_json", "{}", String),
+                            ("status", "Ok", String),
+                            ("duration", duration_with_overhead, i64),
+                            ("compute_units", cu, i64),
+                        );
+                        */
+                        s.send((ss, h[0..10].into_iter().copied().collect::<Vec<_>>()))
+                            .unwrap();
+                        i += 1;
+                    }
+                })
+                .unwrap()
+        })
+        .collect::<Vec<_>>();
+
+    //joins.push(p);
+
+    joins.push(pc);
+    joins.into_iter().for_each(|j| j.join().unwrap());
+}

From efd10117d0999f3c0ab0304af3428581e480e9e0 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 5 Aug 2022 21:51:41 +0900
Subject: [PATCH 0005/3199] save

---
 scheduler/Cargo.toml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/Cargo.toml b/scheduler/Cargo.toml
index e617b893059139..74dc5af5d22d3c 100644
--- a/scheduler/Cargo.toml
+++ b/scheduler/Cargo.toml
@@ -3,7 +3,7 @@ authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 edition = "2021"
 name = "solana-scheduler"
 description = "The solana scheduler"
-version = "1.11.4"
+version = "1.11.5"
 repository = "https://github.com/solana-labs/solana"
 license = "Apache-2.0"
 homepage = "https://solana.com/"

From bb4f8c1d36d20fcbee1488b01ee60325b18f7c11 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 5 Aug 2022 21:52:01 +0900
Subject: [PATCH 0006/3199] save

---
 scheduler/Cargo.toml | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/scheduler/Cargo.toml b/scheduler/Cargo.toml
index 74dc5af5d22d3c..ae9f9b292419b6 100644
--- a/scheduler/Cargo.toml
+++ b/scheduler/Cargo.toml
@@ -14,13 +14,13 @@ publish = true
 atomic_enum = "0.1.1"
 crossbeam-channel = "0.5.5"
 log = "0.4.17"
-solana-entry = { path = "../entry", version = "=1.11.4" }
-solana-runtime = { path = "../runtime", version = "=1.11.4" }
-solana-sdk = { path = "../sdk", version = "=1.11.4" }
-solana-logger = { path = "../logger", version = "=1.11.4" }
-solana-version = { path = "../version", version = "=1.11.4" }
-solana-metrics = { path = "../metrics", version = "=1.11.4" }
-solana-measure = { path = "../measure", version = "=1.11.4" }
+solana-entry = { path = "../entry", version = "=1.11.5" }
+solana-runtime = { path = "../runtime", version = "=1.11.5" }
+solana-sdk = { path = "../sdk", version = "=1.11.5" }
+solana-logger = { path = "../logger", version = "=1.11.5" }
+solana-version = { path = "../version", version = "=1.11.5" }
+solana-metrics = { path = "../metrics", version = "=1.11.5" }
+solana-measure = { path = "../measure", version = "=1.11.5" }
 sha2 = { version = "0.10.2" }
 rand = "0.7.0"
 

From 089be4ade14bc32ccb763a7f225127f97da518bd Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 5 Aug 2022 22:05:13 +0900
Subject: [PATCH 0007/3199] save

---
 Cargo.lock | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/Cargo.lock b/Cargo.lock
index ec5641cbf62a1b..54cffd2376424f 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -254,9 +254,9 @@ version = "0.1.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "6a57588a20357c74ee9036656b7fad5c54328ee8657c6ff29ba31fd5ea503a1c"
 dependencies = [
- "proc-macro2 1.0.38",
+ "proc-macro2 1.0.41",
  "quote 1.0.18",
- "syn 1.0.93",
+ "syn 1.0.98",
 ]
 
 [[package]]
@@ -5525,8 +5525,8 @@ dependencies = [
  "solana-measure",
  "solana-metrics",
  "solana-runtime",
- "solana-sdk 1.11.5",
  "solana-scheduler",
+ "solana-sdk 1.11.5",
  "solana-stake-program",
  "solana-storage-bigtable",
  "solana-transaction-status",
@@ -6058,7 +6058,7 @@ dependencies = [
 
 [[package]]
 name = "solana-scheduler"
-version = "1.11.4"
+version = "1.11.5"
 dependencies = [
  "atomic_enum",
  "crossbeam-channel",
@@ -6066,11 +6066,11 @@ dependencies = [
  "rand 0.7.3",
  "sha2 0.10.2",
  "solana-entry",
- "solana-logger 1.11.4",
+ "solana-logger 1.11.5",
  "solana-measure",
  "solana-metrics",
  "solana-runtime",
- "solana-sdk 1.11.4",
+ "solana-sdk 1.11.5",
  "solana-version",
 ]
 

From 717273e91e98920788884fd6862db351d98ae945 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 5 Aug 2022 22:09:54 +0900
Subject: [PATCH 0008/3199] save

---
 transaction-scheduler-bench/Cargo.toml | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/transaction-scheduler-bench/Cargo.toml b/transaction-scheduler-bench/Cargo.toml
index d963b62ac5bfaf..62cacad52fe7c1 100644
--- a/transaction-scheduler-bench/Cargo.toml
+++ b/transaction-scheduler-bench/Cargo.toml
@@ -23,4 +23,5 @@ solana-logger = { path = "../logger" }
 solana-measure = { path = "../measure" }
 solana-perf = { path = "../perf" }
 solana-runtime = { path = "../runtime" }
-solana-sdk = { path = "../sdk" }
\ No newline at end of file
+solana-schedule = { path = "../schedule" }
+solana-sdk = { path = "../sdk" }

From 4bc00f7ff9e1754313c2ed2f963157602c674755 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 5 Aug 2022 22:10:15 +0900
Subject: [PATCH 0009/3199] save

---
 transaction-scheduler-bench/Cargo.toml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/transaction-scheduler-bench/Cargo.toml b/transaction-scheduler-bench/Cargo.toml
index 62cacad52fe7c1..1262c6e3f98468 100644
--- a/transaction-scheduler-bench/Cargo.toml
+++ b/transaction-scheduler-bench/Cargo.toml
@@ -23,5 +23,5 @@ solana-logger = { path = "../logger" }
 solana-measure = { path = "../measure" }
 solana-perf = { path = "../perf" }
 solana-runtime = { path = "../runtime" }
-solana-schedule = { path = "../schedule" }
+solana-scheduler = { path = "../scheduler" }
 solana-sdk = { path = "../sdk" }

From 0e6070ae7de1c9779dd1f65db4ce749b631c91a8 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 5 Aug 2022 22:11:35 +0900
Subject: [PATCH 0010/3199] save

---
 Cargo.lock                              |  1 +
 transaction-scheduler-bench/src/main.rs | 12 +++++++++++-
 2 files changed, 12 insertions(+), 1 deletion(-)

diff --git a/Cargo.lock b/Cargo.lock
index 390792c5e3f40d..2e35573b8daa73 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -6458,6 +6458,7 @@ dependencies = [
  "solana-measure",
  "solana-perf",
  "solana-runtime",
+ "solana-scheduler",
  "solana-sdk 1.11.5",
 ]
 
diff --git a/transaction-scheduler-bench/src/main.rs b/transaction-scheduler-bench/src/main.rs
index acfb1d172ef434..ae65eb4ecc6985 100644
--- a/transaction-scheduler-bench/src/main.rs
+++ b/transaction-scheduler-bench/src/main.rs
@@ -114,6 +114,16 @@ struct PacketSendingConfig {
     num_write_locks_per_tx: usize,
 }
 
+fn spawn_unified_scheduler(
+        packet_batch_receiver: Receiver<PacketBatchMessage>,
+        transaction_batch_senders: Vec<Sender<TransactionBatchMessage>>,
+        completed_transaction_receiver: Receiver<CompletedTransactionMessage>,
+        bank_forks: Arc<RwLock<BankForks>>,
+        max_batch_size: usize,
+        exit: Arc<AtomicBool>,
+        ) {
+}
+
 fn main() {
     solana_logger::setup_with_default("INFO");
 
@@ -142,7 +152,7 @@ fn main() {
     let exit = Arc::new(AtomicBool::new(false));
 
     // Spawns and runs the scheduler thread
-    let scheduler_handle = TransactionScheduler::spawn_scheduler(
+    let scheduler_handle = spawn_unified_scheduler(
         packet_batch_receiver,
         transaction_batch_senders,
         completed_transaction_receiver,

From faf9c512721da404c6da7e95fa25b1460a5223f8 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 5 Aug 2022 22:13:42 +0900
Subject: [PATCH 0011/3199] save

---
 core/src/transaction_scheduler.rs       |  8 ++++----
 transaction-scheduler-bench/src/main.rs | 14 +++++++-------
 2 files changed, 11 insertions(+), 11 deletions(-)

diff --git a/core/src/transaction_scheduler.rs b/core/src/transaction_scheduler.rs
index fed4c2edb6ba75..80f63083c48166 100644
--- a/core/src/transaction_scheduler.rs
+++ b/core/src/transaction_scheduler.rs
@@ -113,10 +113,10 @@ impl TransactionPriority {
     }
 }
 
-type PacketBatchMessage = Vec<PacketBatch>;
-type TransactionMessage = Box<SanitizedTransaction>;
-type CompletedTransactionMessage = (usize, TransactionMessage);
-type TransactionBatchMessage = Vec<TransactionMessage>;
+pub type PacketBatchMessage = Vec<PacketBatch>;
+pub type TransactionMessage = Box<SanitizedTransaction>;
+pub type CompletedTransactionMessage = (usize, TransactionMessage);
+pub type TransactionBatchMessage = Vec<TransactionMessage>;
 
 /// Separate packet deserialization and ordering
 struct PacketBatchHandler {
diff --git a/transaction-scheduler-bench/src/main.rs b/transaction-scheduler-bench/src/main.rs
index ae65eb4ecc6985..f502884604dde2 100644
--- a/transaction-scheduler-bench/src/main.rs
+++ b/transaction-scheduler-bench/src/main.rs
@@ -115,13 +115,13 @@ struct PacketSendingConfig {
 }
 
 fn spawn_unified_scheduler(
-        packet_batch_receiver: Receiver<PacketBatchMessage>,
-        transaction_batch_senders: Vec<Sender<TransactionBatchMessage>>,
-        completed_transaction_receiver: Receiver<CompletedTransactionMessage>,
-        bank_forks: Arc<RwLock<BankForks>>,
-        max_batch_size: usize,
-        exit: Arc<AtomicBool>,
-        ) {
+    packet_batch_receiver: Receiver<PacketBatchMessage>,
+    transaction_batch_senders: Vec<Sender<TransactionBatchMessage>>,
+    completed_transaction_receiver: Receiver<CompletedTransactionMessage>,
+    bank_forks: Arc<RwLock<BankForks>>,
+    max_batch_size: usize,
+    exit: Arc<AtomicBool>,
+) {
 }
 
 fn main() {

From 47e3e31428cb74bb8c46d79e1a3321c39788701a Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 5 Aug 2022 22:22:31 +0900
Subject: [PATCH 0012/3199] save

---
 transaction-scheduler-bench/src/main.rs | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/transaction-scheduler-bench/src/main.rs b/transaction-scheduler-bench/src/main.rs
index f502884604dde2..a53ef98d6e9090 100644
--- a/transaction-scheduler-bench/src/main.rs
+++ b/transaction-scheduler-bench/src/main.rs
@@ -77,10 +77,6 @@ struct Args {
     high_conflict_sender: usize,
 }
 
-/// Some convenient type aliases
-type TransactionMessage = Box<SanitizedTransaction>;
-type CompletedTransactionMessage = (usize, TransactionMessage); // thread index and transaction message
-type TransactionBatchMessage = Vec<TransactionMessage>;
 
 #[derive(Debug, Default)]
 struct TransactionSchedulerBenchMetrics {

From 8dfa295151f9de914597acb0b6c4b0aeb381db44 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 5 Aug 2022 22:22:57 +0900
Subject: [PATCH 0013/3199] save

---
 transaction-scheduler-bench/src/main.rs | 14 ++++----------
 1 file changed, 4 insertions(+), 10 deletions(-)

diff --git a/transaction-scheduler-bench/src/main.rs b/transaction-scheduler-bench/src/main.rs
index a53ef98d6e9090..147d45600702b3 100644
--- a/transaction-scheduler-bench/src/main.rs
+++ b/transaction-scheduler-bench/src/main.rs
@@ -77,6 +77,10 @@ struct Args {
     high_conflict_sender: usize,
 }
 
+/// Some convenient type aliases
+type TransactionMessage = Box<SanitizedTransaction>;
+type CompletedTransactionMessage = (usize, TransactionMessage); // thread index and transaction message
+type TransactionBatchMessage = Vec<TransactionMessage>;
 
 #[derive(Debug, Default)]
 struct TransactionSchedulerBenchMetrics {
@@ -110,16 +114,6 @@ struct PacketSendingConfig {
     num_write_locks_per_tx: usize,
 }
 
-fn spawn_unified_scheduler(
-    packet_batch_receiver: Receiver<PacketBatchMessage>,
-    transaction_batch_senders: Vec<Sender<TransactionBatchMessage>>,
-    completed_transaction_receiver: Receiver<CompletedTransactionMessage>,
-    bank_forks: Arc<RwLock<BankForks>>,
-    max_batch_size: usize,
-    exit: Arc<AtomicBool>,
-) {
-}
-
 fn main() {
     solana_logger::setup_with_default("INFO");
 

From 933721bc0b91506f67d56d213925e07e457f46db Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 5 Aug 2022 22:23:53 +0900
Subject: [PATCH 0014/3199] save

---
 transaction-scheduler-bench/src/main.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/transaction-scheduler-bench/src/main.rs b/transaction-scheduler-bench/src/main.rs
index 147d45600702b3..acfb1d172ef434 100644
--- a/transaction-scheduler-bench/src/main.rs
+++ b/transaction-scheduler-bench/src/main.rs
@@ -142,7 +142,7 @@ fn main() {
     let exit = Arc::new(AtomicBool::new(false));
 
     // Spawns and runs the scheduler thread
-    let scheduler_handle = spawn_unified_scheduler(
+    let scheduler_handle = TransactionScheduler::spawn_scheduler(
         packet_batch_receiver,
         transaction_batch_senders,
         completed_transaction_receiver,

From 4f6ab39bd6e490fd380d66738f87bb3feb894ef0 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 5 Aug 2022 22:31:52 +0900
Subject: [PATCH 0015/3199] save

---
 transaction-scheduler-bench/src/main.rs | 10 +++++++++-
 1 file changed, 9 insertions(+), 1 deletion(-)

diff --git a/transaction-scheduler-bench/src/main.rs b/transaction-scheduler-bench/src/main.rs
index acfb1d172ef434..3a12ae58c77228 100644
--- a/transaction-scheduler-bench/src/main.rs
+++ b/transaction-scheduler-bench/src/main.rs
@@ -467,7 +467,15 @@ fn build_packet(
         &[sending_keypair],
         *blockhash,
     ));
-    Packet::from_data(None, &versioned_transaction).unwrap()
+
+    SanitizedTransaction::try_create(
+        versioned_transaction,
+        MessageHash::Compute,
+        None,
+        SimpleAddressLoader::Disabled,
+        true, // require_static_program_ids
+    )
+    .unwrap()
 }
 
 fn build_accounts(num_accounts: usize) -> Vec<Keypair> {

From 197621af8a5ba30c5fa27011c1f021662c8e7361 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 5 Aug 2022 22:32:18 +0900
Subject: [PATCH 0016/3199] save

---
 transaction-scheduler-bench/src/main.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/transaction-scheduler-bench/src/main.rs b/transaction-scheduler-bench/src/main.rs
index 3a12ae58c77228..cab0defd4465eb 100644
--- a/transaction-scheduler-bench/src/main.rs
+++ b/transaction-scheduler-bench/src/main.rs
@@ -472,7 +472,7 @@ fn build_packet(
         versioned_transaction,
         MessageHash::Compute,
         None,
-        SimpleAddressLoader::Disabled,
+        solana_sdk::transaction::SimpleAddressLoader::Disabled,
         true, // require_static_program_ids
     )
     .unwrap()

From a11c9eed246586ec05995a0a40b1a5142284a2a8 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 5 Aug 2022 22:32:35 +0900
Subject: [PATCH 0017/3199] save

---
 transaction-scheduler-bench/src/main.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/transaction-scheduler-bench/src/main.rs b/transaction-scheduler-bench/src/main.rs
index cab0defd4465eb..6e09929ff02f5a 100644
--- a/transaction-scheduler-bench/src/main.rs
+++ b/transaction-scheduler-bench/src/main.rs
@@ -470,7 +470,7 @@ fn build_packet(
 
     SanitizedTransaction::try_create(
         versioned_transaction,
-        MessageHash::Compute,
+        solana_sdk::transaction::MessageHash::Compute,
         None,
         solana_sdk::transaction::SimpleAddressLoader::Disabled,
         true, // require_static_program_ids

From 7f731c6b692afe6b4f24d5c6b63be071792ac12e Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 5 Aug 2022 22:32:47 +0900
Subject: [PATCH 0018/3199] save

---
 transaction-scheduler-bench/src/main.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/transaction-scheduler-bench/src/main.rs b/transaction-scheduler-bench/src/main.rs
index 6e09929ff02f5a..84df10b33712a6 100644
--- a/transaction-scheduler-bench/src/main.rs
+++ b/transaction-scheduler-bench/src/main.rs
@@ -445,7 +445,7 @@ fn build_packet(
     num_accounts: usize,
     accounts: &[Keypair],
     blockhash: &Hash,
-) -> Packet {
+) -> SanitizedTransaction {
     let get_random_account = || &accounts[rand::thread_rng().gen_range(0..num_accounts)];
     let sending_keypair = get_random_account();
 

From f10536c0bbb07edba2ccb744790ed4427f4c5b48 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 5 Aug 2022 22:33:06 +0900
Subject: [PATCH 0019/3199] save

---
 transaction-scheduler-bench/src/main.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/transaction-scheduler-bench/src/main.rs b/transaction-scheduler-bench/src/main.rs
index 84df10b33712a6..aa43d345ff9c17 100644
--- a/transaction-scheduler-bench/src/main.rs
+++ b/transaction-scheduler-bench/src/main.rs
@@ -432,7 +432,7 @@ fn build_packet_batch(
     num_accounts: usize,
     accounts: &[Keypair],
     blockhash: &Hash,
-) -> PacketBatch {
+) -> Vec<SanitizedTransaction> {
     PacketBatch::new(
         (0..config.packets_per_batch)
             .map(|_| build_packet(config, num_accounts, accounts, blockhash))

From 14d013c18abacfb0663b1667444e4bf5f31c5baf Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 5 Aug 2022 22:33:31 +0900
Subject: [PATCH 0020/3199] save

---
 transaction-scheduler-bench/src/main.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/transaction-scheduler-bench/src/main.rs b/transaction-scheduler-bench/src/main.rs
index aa43d345ff9c17..4460fe4ac8fa8a 100644
--- a/transaction-scheduler-bench/src/main.rs
+++ b/transaction-scheduler-bench/src/main.rs
@@ -421,7 +421,7 @@ fn build_packet_batches(
     num_accounts: usize,
     accounts: &[Keypair],
     blockhash: &Hash,
-) -> Vec<PacketBatch> {
+) -> Vec<Vec<SanitizedTransaction>> {
     (0..config.batches_per_msg)
         .map(|_| build_packet_batch(config, num_accounts, accounts, blockhash))
         .collect()

From 21b771ac76fd00a3c706ebbd66a656041fd127ab Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 5 Aug 2022 22:34:05 +0900
Subject: [PATCH 0021/3199] save

---
 transaction-scheduler-bench/src/main.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/transaction-scheduler-bench/src/main.rs b/transaction-scheduler-bench/src/main.rs
index 4460fe4ac8fa8a..f03238e49ffc09 100644
--- a/transaction-scheduler-bench/src/main.rs
+++ b/transaction-scheduler-bench/src/main.rs
@@ -377,7 +377,7 @@ fn send_packets(
     metrics: Arc<TransactionSchedulerBenchMetrics>,
     num_accounts: usize,
     accounts: Arc<Vec<Keypair>>,
-    packet_batch_sender: Sender<Vec<PacketBatch>>,
+    packet_batch_sender: Sender<Vec<Vec<SanitizedTransaction>>>,
     config: Arc<PacketSendingConfig>,
     duration: Duration,
     exit: Arc<AtomicBool>,

From 338c4d1f9fea6dd9daf25d82eaab7ffb4681f425 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 5 Aug 2022 22:34:25 +0900
Subject: [PATCH 0022/3199] save

---
 transaction-scheduler-bench/src/main.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/transaction-scheduler-bench/src/main.rs b/transaction-scheduler-bench/src/main.rs
index f03238e49ffc09..e717bef8588b01 100644
--- a/transaction-scheduler-bench/src/main.rs
+++ b/transaction-scheduler-bench/src/main.rs
@@ -355,7 +355,7 @@ fn spawn_packet_sender(
     metrics: Arc<TransactionSchedulerBenchMetrics>,
     num_accounts: usize,
     accounts: Arc<Vec<Keypair>>,
-    packet_batch_sender: Sender<Vec<PacketBatch>>,
+    packet_batch_sender: Sender<Vec<Vec<SanitizedTransaction>>>,
     config: Arc<PacketSendingConfig>,
     duration: Duration,
     exit: Arc<AtomicBool>,

From 45c2a326cc0308cd10a8ae9be049baae21b40338 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 5 Aug 2022 22:34:45 +0900
Subject: [PATCH 0023/3199] save

---
 transaction-scheduler-bench/src/main.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/transaction-scheduler-bench/src/main.rs b/transaction-scheduler-bench/src/main.rs
index e717bef8588b01..2668fe250a7a71 100644
--- a/transaction-scheduler-bench/src/main.rs
+++ b/transaction-scheduler-bench/src/main.rs
@@ -326,7 +326,7 @@ fn spawn_packet_senders(
     metrics: Arc<TransactionSchedulerBenchMetrics>,
     high_conflict_sender: usize,
     accounts: Arc<Vec<Keypair>>,
-    packet_batch_sender: Sender<Vec<PacketBatch>>,
+    packet_batch_sender: Sender<Vec<Vec<SanitizedTransaction>>>,
     config: Arc<PacketSendingConfig>,
     duration: Duration,
     exit: Arc<AtomicBool>,

From fb8c35cddc1ac9cc512b51c75f45491403484b83 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 5 Aug 2022 22:36:00 +0900
Subject: [PATCH 0024/3199] save

---
 transaction-scheduler-bench/src/main.rs | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/transaction-scheduler-bench/src/main.rs b/transaction-scheduler-bench/src/main.rs
index 2668fe250a7a71..22eef1daec5e01 100644
--- a/transaction-scheduler-bench/src/main.rs
+++ b/transaction-scheduler-bench/src/main.rs
@@ -114,6 +114,9 @@ struct PacketSendingConfig {
     num_write_locks_per_tx: usize,
 }
 
+fn spawn_unified_scheduler() {
+}
+
 fn main() {
     solana_logger::setup_with_default("INFO");
 
@@ -142,7 +145,7 @@ fn main() {
     let exit = Arc::new(AtomicBool::new(false));
 
     // Spawns and runs the scheduler thread
-    let scheduler_handle = TransactionScheduler::spawn_scheduler(
+    let scheduler_handle = spawn_unified_scheduler(
         packet_batch_receiver,
         transaction_batch_senders,
         completed_transaction_receiver,

From dabb28f57a984bc94f7acfa6ef2d0624e27cbb11 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 5 Aug 2022 22:36:40 +0900
Subject: [PATCH 0025/3199] save

---
 transaction-scheduler-bench/src/main.rs | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/transaction-scheduler-bench/src/main.rs b/transaction-scheduler-bench/src/main.rs
index 22eef1daec5e01..d835b824ef5fbc 100644
--- a/transaction-scheduler-bench/src/main.rs
+++ b/transaction-scheduler-bench/src/main.rs
@@ -114,7 +114,14 @@ struct PacketSendingConfig {
     num_write_locks_per_tx: usize,
 }
 
-fn spawn_unified_scheduler() {
+fn spawn_unified_scheduler(
+        packet_batch_receiver: Receiver<PacketBatchMessage>,
+        transaction_batch_senders: Vec<Sender<TransactionBatchMessage>>,
+        completed_transaction_receiver: Receiver<CompletedTransactionMessage>,
+        bank_forks: Arc<RwLock<BankForks>>,
+        max_batch_size: usize,
+        exit: Arc<AtomicBool>,
+) {
 }
 
 fn main() {

From 4d8eb4e9382aba5902d9a3d3cffc8ad3f300097c Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 5 Aug 2022 22:37:08 +0900
Subject: [PATCH 0026/3199] save

---
 transaction-scheduler-bench/src/main.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/transaction-scheduler-bench/src/main.rs b/transaction-scheduler-bench/src/main.rs
index d835b824ef5fbc..b7282cb188e8d2 100644
--- a/transaction-scheduler-bench/src/main.rs
+++ b/transaction-scheduler-bench/src/main.rs
@@ -115,7 +115,7 @@ struct PacketSendingConfig {
 }
 
 fn spawn_unified_scheduler(
-        packet_batch_receiver: Receiver<PacketBatchMessage>,
+        packet_batch_receiver: Receiver<Vec<Vec<SanitizedTransaction>>>,
         transaction_batch_senders: Vec<Sender<TransactionBatchMessage>>,
         completed_transaction_receiver: Receiver<CompletedTransactionMessage>,
         bank_forks: Arc<RwLock<BankForks>>,

From 510583f28c2c16eaec62f143a76166600de07d3a Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 5 Aug 2022 22:37:45 +0900
Subject: [PATCH 0027/3199] save

---
 transaction-scheduler-bench/src/main.rs | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/transaction-scheduler-bench/src/main.rs b/transaction-scheduler-bench/src/main.rs
index b7282cb188e8d2..7501a80581506e 100644
--- a/transaction-scheduler-bench/src/main.rs
+++ b/transaction-scheduler-bench/src/main.rs
@@ -121,7 +121,8 @@ fn spawn_unified_scheduler(
         bank_forks: Arc<RwLock<BankForks>>,
         max_batch_size: usize,
         exit: Arc<AtomicBool>,
-) {
+) -> usize {
+    std::thread::spawn(|| {})
 }
 
 fn main() {

From fa61ef423324e074eb43c299744938be4792e53e Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 5 Aug 2022 22:38:08 +0900
Subject: [PATCH 0028/3199] save

---
 transaction-scheduler-bench/src/main.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/transaction-scheduler-bench/src/main.rs b/transaction-scheduler-bench/src/main.rs
index 7501a80581506e..ce3f2f0fe6bad2 100644
--- a/transaction-scheduler-bench/src/main.rs
+++ b/transaction-scheduler-bench/src/main.rs
@@ -121,7 +121,7 @@ fn spawn_unified_scheduler(
         bank_forks: Arc<RwLock<BankForks>>,
         max_batch_size: usize,
         exit: Arc<AtomicBool>,
-) -> usize {
+) -> JoinHandle<()> {
     std::thread::spawn(|| {})
 }
 

From b6deb4d6a3af98fcee62411433f23348ab93c37a Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 5 Aug 2022 22:38:24 +0900
Subject: [PATCH 0029/3199] save

---
 transaction-scheduler-bench/src/main.rs | 8 +++-----
 1 file changed, 3 insertions(+), 5 deletions(-)

diff --git a/transaction-scheduler-bench/src/main.rs b/transaction-scheduler-bench/src/main.rs
index ce3f2f0fe6bad2..263cb6bd87fb59 100644
--- a/transaction-scheduler-bench/src/main.rs
+++ b/transaction-scheduler-bench/src/main.rs
@@ -444,11 +444,9 @@ fn build_packet_batch(
     accounts: &[Keypair],
     blockhash: &Hash,
 ) -> Vec<SanitizedTransaction> {
-    PacketBatch::new(
-        (0..config.packets_per_batch)
-            .map(|_| build_packet(config, num_accounts, accounts, blockhash))
-            .collect(),
-    )
+    (0..config.packets_per_batch)
+        .map(|_| build_packet(config, num_accounts, accounts, blockhash))
+        .collect(),
 }
 
 fn build_packet(

From 5c432c3fdcd45a348a87bcbc88588cd3435b1fc0 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 5 Aug 2022 22:38:30 +0900
Subject: [PATCH 0030/3199] save

---
 transaction-scheduler-bench/src/main.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/transaction-scheduler-bench/src/main.rs b/transaction-scheduler-bench/src/main.rs
index 263cb6bd87fb59..6d25458cacaaa8 100644
--- a/transaction-scheduler-bench/src/main.rs
+++ b/transaction-scheduler-bench/src/main.rs
@@ -446,7 +446,7 @@ fn build_packet_batch(
 ) -> Vec<SanitizedTransaction> {
     (0..config.packets_per_batch)
         .map(|_| build_packet(config, num_accounts, accounts, blockhash))
-        .collect(),
+        .collect()
 }
 
 fn build_packet(

From 115049d050ebd2fb62248c197d4629f70291fecd Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 5 Aug 2022 22:41:52 +0900
Subject: [PATCH 0031/3199] save

---
 transaction-scheduler-bench/src/main.rs | 17 +++++++++--------
 1 file changed, 9 insertions(+), 8 deletions(-)

diff --git a/transaction-scheduler-bench/src/main.rs b/transaction-scheduler-bench/src/main.rs
index 6d25458cacaaa8..f3e220ffa3f203 100644
--- a/transaction-scheduler-bench/src/main.rs
+++ b/transaction-scheduler-bench/src/main.rs
@@ -78,7 +78,8 @@ struct Args {
 }
 
 /// Some convenient type aliases
-type TransactionMessage = Box<SanitizedTransaction>;
+type PreprocessedTransaction = SanitizedTransaction
+type TransactionMessage = Box<PreprocessedTransaction>;
 type CompletedTransactionMessage = (usize, TransactionMessage); // thread index and transaction message
 type TransactionBatchMessage = Vec<TransactionMessage>;
 
@@ -115,7 +116,7 @@ struct PacketSendingConfig {
 }
 
 fn spawn_unified_scheduler(
-        packet_batch_receiver: Receiver<Vec<Vec<SanitizedTransaction>>>,
+        packet_batch_receiver: Receiver<Vec<Vec<PreprocessedTransaction>>>,
         transaction_batch_senders: Vec<Sender<TransactionBatchMessage>>,
         completed_transaction_receiver: Receiver<CompletedTransactionMessage>,
         bank_forks: Arc<RwLock<BankForks>>,
@@ -337,7 +338,7 @@ fn spawn_packet_senders(
     metrics: Arc<TransactionSchedulerBenchMetrics>,
     high_conflict_sender: usize,
     accounts: Arc<Vec<Keypair>>,
-    packet_batch_sender: Sender<Vec<Vec<SanitizedTransaction>>>,
+    packet_batch_sender: Sender<Vec<Vec<PreprocessedTransaction>>>,
     config: Arc<PacketSendingConfig>,
     duration: Duration,
     exit: Arc<AtomicBool>,
@@ -366,7 +367,7 @@ fn spawn_packet_sender(
     metrics: Arc<TransactionSchedulerBenchMetrics>,
     num_accounts: usize,
     accounts: Arc<Vec<Keypair>>,
-    packet_batch_sender: Sender<Vec<Vec<SanitizedTransaction>>>,
+    packet_batch_sender: Sender<Vec<Vec<PreprocessedTransaction>>>,
     config: Arc<PacketSendingConfig>,
     duration: Duration,
     exit: Arc<AtomicBool>,
@@ -388,7 +389,7 @@ fn send_packets(
     metrics: Arc<TransactionSchedulerBenchMetrics>,
     num_accounts: usize,
     accounts: Arc<Vec<Keypair>>,
-    packet_batch_sender: Sender<Vec<Vec<SanitizedTransaction>>>,
+    packet_batch_sender: Sender<Vec<Vec<PreprocessedTransaction>>>,
     config: Arc<PacketSendingConfig>,
     duration: Duration,
     exit: Arc<AtomicBool>,
@@ -432,7 +433,7 @@ fn build_packet_batches(
     num_accounts: usize,
     accounts: &[Keypair],
     blockhash: &Hash,
-) -> Vec<Vec<SanitizedTransaction>> {
+) -> Vec<Vec<PreprocessedTransaction>> {
     (0..config.batches_per_msg)
         .map(|_| build_packet_batch(config, num_accounts, accounts, blockhash))
         .collect()
@@ -443,7 +444,7 @@ fn build_packet_batch(
     num_accounts: usize,
     accounts: &[Keypair],
     blockhash: &Hash,
-) -> Vec<SanitizedTransaction> {
+) -> Vec<PreprocessedTransaction> {
     (0..config.packets_per_batch)
         .map(|_| build_packet(config, num_accounts, accounts, blockhash))
         .collect()
@@ -454,7 +455,7 @@ fn build_packet(
     num_accounts: usize,
     accounts: &[Keypair],
     blockhash: &Hash,
-) -> SanitizedTransaction {
+) -> PreprocessedTransaction {
     let get_random_account = || &accounts[rand::thread_rng().gen_range(0..num_accounts)];
     let sending_keypair = get_random_account();
 

From 3d6d769a866b387ff95bafa2c0fb1d3eb1d04739 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 5 Aug 2022 22:42:02 +0900
Subject: [PATCH 0032/3199] save

---
 transaction-scheduler-bench/src/main.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/transaction-scheduler-bench/src/main.rs b/transaction-scheduler-bench/src/main.rs
index f3e220ffa3f203..340a0648ba55b1 100644
--- a/transaction-scheduler-bench/src/main.rs
+++ b/transaction-scheduler-bench/src/main.rs
@@ -78,7 +78,7 @@ struct Args {
 }
 
 /// Some convenient type aliases
-type PreprocessedTransaction = SanitizedTransaction
+type PreprocessedTransaction = SanitizedTransaction;
 type TransactionMessage = Box<PreprocessedTransaction>;
 type CompletedTransactionMessage = (usize, TransactionMessage); // thread index and transaction message
 type TransactionBatchMessage = Vec<TransactionMessage>;

From 94adb97e3c63fd4543d9abc4aea6f751e90ef185 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 5 Aug 2022 22:42:41 +0900
Subject: [PATCH 0033/3199] save

---
 transaction-scheduler-bench/src/main.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/transaction-scheduler-bench/src/main.rs b/transaction-scheduler-bench/src/main.rs
index 340a0648ba55b1..eb9a8ee402848d 100644
--- a/transaction-scheduler-bench/src/main.rs
+++ b/transaction-scheduler-bench/src/main.rs
@@ -78,7 +78,7 @@ struct Args {
 }
 
 /// Some convenient type aliases
-type PreprocessedTransaction = SanitizedTransaction;
+type PreprocessedTransaction = (SanitizedTransaction, Vec<LockAttempt>);
 type TransactionMessage = Box<PreprocessedTransaction>;
 type CompletedTransactionMessage = (usize, TransactionMessage); // thread index and transaction message
 type TransactionBatchMessage = Vec<TransactionMessage>;

From ec9e25ab4b15afa2b448ef7b381294be3569835a Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 5 Aug 2022 22:43:01 +0900
Subject: [PATCH 0034/3199] save

---
 transaction-scheduler-bench/src/main.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/transaction-scheduler-bench/src/main.rs b/transaction-scheduler-bench/src/main.rs
index eb9a8ee402848d..92f104f94f791b 100644
--- a/transaction-scheduler-bench/src/main.rs
+++ b/transaction-scheduler-bench/src/main.rs
@@ -78,7 +78,7 @@ struct Args {
 }
 
 /// Some convenient type aliases
-type PreprocessedTransaction = (SanitizedTransaction, Vec<LockAttempt>);
+type PreprocessedTransaction = (SanitizedTransaction, Vec<solana_scheduler::LockAttempt>);
 type TransactionMessage = Box<PreprocessedTransaction>;
 type CompletedTransactionMessage = (usize, TransactionMessage); // thread index and transaction message
 type TransactionBatchMessage = Vec<TransactionMessage>;

From 6c897177fcd5b7f6842e4722bf484ff8b7ba74ab Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 5 Aug 2022 22:43:28 +0900
Subject: [PATCH 0035/3199] save

---
 transaction-scheduler-bench/src/main.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/transaction-scheduler-bench/src/main.rs b/transaction-scheduler-bench/src/main.rs
index 92f104f94f791b..662c958188e9d4 100644
--- a/transaction-scheduler-bench/src/main.rs
+++ b/transaction-scheduler-bench/src/main.rs
@@ -315,7 +315,7 @@ fn handle_transaction_batch(
 
     let priority_collected = transaction_batch
         .iter()
-        .map(|tx| tx.get_transaction_priority_details().unwrap().priority)
+        .map(|tx| tx.0.get_transaction_priority_details().unwrap().priority)
         .sum();
 
     metrics

From 1f9b26ddddc2a24cf87a4d6951ae057f88a93fb1 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 5 Aug 2022 22:44:35 +0900
Subject: [PATCH 0036/3199] save

---
 transaction-scheduler-bench/src/main.rs | 16 ++++++++++++++--
 1 file changed, 14 insertions(+), 2 deletions(-)

diff --git a/transaction-scheduler-bench/src/main.rs b/transaction-scheduler-bench/src/main.rs
index 662c958188e9d4..651d07df278422 100644
--- a/transaction-scheduler-bench/src/main.rs
+++ b/transaction-scheduler-bench/src/main.rs
@@ -478,14 +478,26 @@ fn build_packet(
         *blockhash,
     ));
 
-    SanitizedTransaction::try_create(
+    let s = SanitizedTransaction::try_create(
         versioned_transaction,
         solana_sdk::transaction::MessageHash::Compute,
         None,
         solana_sdk::transaction::SimpleAddressLoader::Disabled,
         true, // require_static_program_ids
     )
-    .unwrap()
+    .unwrap();
+
+    let locks = sanitized_tx.get_account_locks().unwrap();
+    let writable_lock_iter = locks
+        .writable
+        .iter()
+        .map(|address| LockAttempt::new(**address, RequestedUsage::Writable));
+    let readonly_lock_iter = locks
+        .readonly
+        .iter()
+        .map(|address| LockAttempt::new(**address, RequestedUsage::Readonly));
+    let locks = writable_lock_iter.chain(readonly_lock_iter).collect::<Vec<_>>();
+    (sanitized_tx, locks)
 }
 
 fn build_accounts(num_accounts: usize) -> Vec<Keypair> {

From f190fa447e610a032438b42f458d0d33273ab4b3 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 5 Aug 2022 22:44:49 +0900
Subject: [PATCH 0037/3199] save

---
 transaction-scheduler-bench/src/main.rs | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/transaction-scheduler-bench/src/main.rs b/transaction-scheduler-bench/src/main.rs
index 651d07df278422..c4d9979d64f9b6 100644
--- a/transaction-scheduler-bench/src/main.rs
+++ b/transaction-scheduler-bench/src/main.rs
@@ -497,7 +497,8 @@ fn build_packet(
         .iter()
         .map(|address| LockAttempt::new(**address, RequestedUsage::Readonly));
     let locks = writable_lock_iter.chain(readonly_lock_iter).collect::<Vec<_>>();
-    (sanitized_tx, locks)
+
+    (s, locks)
 }
 
 fn build_accounts(num_accounts: usize) -> Vec<Keypair> {

From 526942ebab2ba5921e6b129af41fc8114906c9a0 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 5 Aug 2022 22:45:07 +0900
Subject: [PATCH 0038/3199] save

---
 transaction-scheduler-bench/src/main.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/transaction-scheduler-bench/src/main.rs b/transaction-scheduler-bench/src/main.rs
index c4d9979d64f9b6..7b35312463405a 100644
--- a/transaction-scheduler-bench/src/main.rs
+++ b/transaction-scheduler-bench/src/main.rs
@@ -491,11 +491,11 @@ fn build_packet(
     let writable_lock_iter = locks
         .writable
         .iter()
-        .map(|address| LockAttempt::new(**address, RequestedUsage::Writable));
+        .map(|address| solana_scheduler::LockAttempt::new(**address, RequestedUsage::Writable));
     let readonly_lock_iter = locks
         .readonly
         .iter()
-        .map(|address| LockAttempt::new(**address, RequestedUsage::Readonly));
+        .map(|address| solana_scheduler::LockAttempt::new(**address, RequestedUsage::Readonly));
     let locks = writable_lock_iter.chain(readonly_lock_iter).collect::<Vec<_>>();
 
     (s, locks)

From 617c346c2c215aa9baf2cb32005fab64e2ed00f7 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 5 Aug 2022 22:45:20 +0900
Subject: [PATCH 0039/3199] save

---
 transaction-scheduler-bench/src/main.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/transaction-scheduler-bench/src/main.rs b/transaction-scheduler-bench/src/main.rs
index 7b35312463405a..41e2d1d2828d3a 100644
--- a/transaction-scheduler-bench/src/main.rs
+++ b/transaction-scheduler-bench/src/main.rs
@@ -478,7 +478,7 @@ fn build_packet(
         *blockhash,
     ));
 
-    let s = SanitizedTransaction::try_create(
+    let sanitized_tx = SanitizedTransaction::try_create(
         versioned_transaction,
         solana_sdk::transaction::MessageHash::Compute,
         None,
@@ -498,7 +498,7 @@ fn build_packet(
         .map(|address| solana_scheduler::LockAttempt::new(**address, RequestedUsage::Readonly));
     let locks = writable_lock_iter.chain(readonly_lock_iter).collect::<Vec<_>>();
 
-    (s, locks)
+    (sanitized_tx, locks)
 }
 
 fn build_accounts(num_accounts: usize) -> Vec<Keypair> {

From effe65322287855f0730594a843c61e9e264fab0 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 5 Aug 2022 22:45:36 +0900
Subject: [PATCH 0040/3199] save

---
 transaction-scheduler-bench/src/main.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/transaction-scheduler-bench/src/main.rs b/transaction-scheduler-bench/src/main.rs
index 41e2d1d2828d3a..85227310f43790 100644
--- a/transaction-scheduler-bench/src/main.rs
+++ b/transaction-scheduler-bench/src/main.rs
@@ -491,11 +491,11 @@ fn build_packet(
     let writable_lock_iter = locks
         .writable
         .iter()
-        .map(|address| solana_scheduler::LockAttempt::new(**address, RequestedUsage::Writable));
+        .map(|address| solana_scheduler::LockAttempt::new(**address, solana_scheduler::RequestedUsage::Writable));
     let readonly_lock_iter = locks
         .readonly
         .iter()
-        .map(|address| solana_scheduler::LockAttempt::new(**address, RequestedUsage::Readonly));
+        .map(|address| solana_scheduler::LockAttempt::new(**address, solana_scheduler::RequestedUsage::Readonly));
     let locks = writable_lock_iter.chain(readonly_lock_iter).collect::<Vec<_>>();
 
     (sanitized_tx, locks)

From 8f40e13721e75a0d404db3492780ac5370ae20ec Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 5 Aug 2022 22:48:38 +0900
Subject: [PATCH 0041/3199] save

---
 transaction-scheduler-bench/src/main.rs | 16 +++++++++++++++-
 1 file changed, 15 insertions(+), 1 deletion(-)

diff --git a/transaction-scheduler-bench/src/main.rs b/transaction-scheduler-bench/src/main.rs
index 85227310f43790..ba534a5a066f26 100644
--- a/transaction-scheduler-bench/src/main.rs
+++ b/transaction-scheduler-bench/src/main.rs
@@ -123,7 +123,21 @@ fn spawn_unified_scheduler(
         max_batch_size: usize,
         exit: Arc<AtomicBool>,
 ) -> JoinHandle<()> {
-    std::thread::spawn(|| {})
+    std::thread::spawn(|| {
+        let mut runnable_queue = TaskQueue::default();
+        let mut contended_queue = TaskQueue::default();
+        let mut address_book = AddressBook::default();
+        ScheduleStage::run(
+            100,
+            &mut runnable_queue,
+            &mut contended_queue,
+            &mut address_book,
+            &packet_batch_receiver,
+            3//&pre_execute_env_sender,
+            4//&post_schedule_env_sender,
+        );
+        })
+    })
 }
 
 fn main() {

From eae6cd7a5de2bfb7dba4ba47ca3cc3d982e27d16 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 5 Aug 2022 22:49:11 +0900
Subject: [PATCH 0042/3199] save

---
 transaction-scheduler-bench/src/main.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/transaction-scheduler-bench/src/main.rs b/transaction-scheduler-bench/src/main.rs
index ba534a5a066f26..17b21627df1b7f 100644
--- a/transaction-scheduler-bench/src/main.rs
+++ b/transaction-scheduler-bench/src/main.rs
@@ -127,6 +127,7 @@ fn spawn_unified_scheduler(
         let mut runnable_queue = TaskQueue::default();
         let mut contended_queue = TaskQueue::default();
         let mut address_book = AddressBook::default();
+
         ScheduleStage::run(
             100,
             &mut runnable_queue,
@@ -136,7 +137,6 @@ fn spawn_unified_scheduler(
             3//&pre_execute_env_sender,
             4//&post_schedule_env_sender,
         );
-        })
     })
 }
 

From 740a9d6f9e15d0bd4bdd96774fe271a93e0f0dae Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 5 Aug 2022 22:49:30 +0900
Subject: [PATCH 0043/3199] save

---
 transaction-scheduler-bench/src/main.rs | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/transaction-scheduler-bench/src/main.rs b/transaction-scheduler-bench/src/main.rs
index 17b21627df1b7f..699e2e70fba82d 100644
--- a/transaction-scheduler-bench/src/main.rs
+++ b/transaction-scheduler-bench/src/main.rs
@@ -124,11 +124,11 @@ fn spawn_unified_scheduler(
         exit: Arc<AtomicBool>,
 ) -> JoinHandle<()> {
     std::thread::spawn(|| {
-        let mut runnable_queue = TaskQueue::default();
-        let mut contended_queue = TaskQueue::default();
-        let mut address_book = AddressBook::default();
+        let mut runnable_queue = solana_scheduler::TaskQueue::default();
+        let mut contended_queue = solana_scheduler::TaskQueue::default();
+        let mut address_book = solana_scheduler::AddressBook::default();
 
-        ScheduleStage::run(
+        solana_scheduler::ScheduleStage::run(
             100,
             &mut runnable_queue,
             &mut contended_queue,

From cb73064b41f42ac4e373d7adccb780750b7bb600 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 5 Aug 2022 22:49:42 +0900
Subject: [PATCH 0044/3199] save

---
 transaction-scheduler-bench/src/main.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/transaction-scheduler-bench/src/main.rs b/transaction-scheduler-bench/src/main.rs
index 699e2e70fba82d..cfaf8d0d44b366 100644
--- a/transaction-scheduler-bench/src/main.rs
+++ b/transaction-scheduler-bench/src/main.rs
@@ -134,8 +134,8 @@ fn spawn_unified_scheduler(
             &mut contended_queue,
             &mut address_book,
             &packet_batch_receiver,
-            3//&pre_execute_env_sender,
-            4//&post_schedule_env_sender,
+            3, //&pre_execute_env_sender,
+            4, //&post_schedule_env_sender,
         );
     })
 }

From 67f1b05833e87b13ca7401bc8697b582e5d05914 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 5 Aug 2022 22:51:07 +0900
Subject: [PATCH 0045/3199] save

---
 transaction-scheduler-bench/src/main.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/transaction-scheduler-bench/src/main.rs b/transaction-scheduler-bench/src/main.rs
index cfaf8d0d44b366..cf591d0fd64351 100644
--- a/transaction-scheduler-bench/src/main.rs
+++ b/transaction-scheduler-bench/src/main.rs
@@ -436,7 +436,7 @@ fn send_packets(
             packet_batches.iter().map(|pb| pb.len()).sum(),
             Ordering::Relaxed,
         );
-        let _ = packet_batch_sender.send(packet_batches);
+        let _ = packet_batch_sender.send(MultiplexedPayload::FromPreviousBatched(packet_batches));
 
         std::thread::sleep(loop_duration.saturating_sub(packet_build_time.as_duration()));
     }

From 07055337d0b33842e2a9529f5917645983b9972c Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 5 Aug 2022 22:51:18 +0900
Subject: [PATCH 0046/3199] save

---
 transaction-scheduler-bench/src/main.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/transaction-scheduler-bench/src/main.rs b/transaction-scheduler-bench/src/main.rs
index cf591d0fd64351..58efb6d10ffd81 100644
--- a/transaction-scheduler-bench/src/main.rs
+++ b/transaction-scheduler-bench/src/main.rs
@@ -436,7 +436,7 @@ fn send_packets(
             packet_batches.iter().map(|pb| pb.len()).sum(),
             Ordering::Relaxed,
         );
-        let _ = packet_batch_sender.send(MultiplexedPayload::FromPreviousBatched(packet_batches));
+        let _ = packet_batch_sender.send(solana_scheduler::MultiplexedPayload::FromPreviousBatched(packet_batches));
 
         std::thread::sleep(loop_duration.saturating_sub(packet_build_time.as_duration()));
     }

From 68f57e8ed38ac486bc438ab911c68bd5951057ff Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 5 Aug 2022 22:52:24 +0900
Subject: [PATCH 0047/3199] save

---
 transaction-scheduler-bench/src/main.rs | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/transaction-scheduler-bench/src/main.rs b/transaction-scheduler-bench/src/main.rs
index 58efb6d10ffd81..2fd571aaa18882 100644
--- a/transaction-scheduler-bench/src/main.rs
+++ b/transaction-scheduler-bench/src/main.rs
@@ -82,6 +82,7 @@ type PreprocessedTransaction = (SanitizedTransaction, Vec<solana_scheduler::Lock
 type TransactionMessage = Box<PreprocessedTransaction>;
 type CompletedTransactionMessage = (usize, TransactionMessage); // thread index and transaction message
 type TransactionBatchMessage = Vec<TransactionMessage>;
+type BatchSenderMessage = Vec<Vec<PreprocessedTransaction>>;
 
 #[derive(Debug, Default)]
 struct TransactionSchedulerBenchMetrics {
@@ -403,7 +404,7 @@ fn send_packets(
     metrics: Arc<TransactionSchedulerBenchMetrics>,
     num_accounts: usize,
     accounts: Arc<Vec<Keypair>>,
-    packet_batch_sender: Sender<Vec<Vec<PreprocessedTransaction>>>,
+    packet_batch_sender: Sender<BatchSenderMessage>,
     config: Arc<PacketSendingConfig>,
     duration: Duration,
     exit: Arc<AtomicBool>,

From 73d5749eec986ce12fd72f890f283075f955b759 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 5 Aug 2022 22:52:45 +0900
Subject: [PATCH 0048/3199] save

---
 transaction-scheduler-bench/src/main.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/transaction-scheduler-bench/src/main.rs b/transaction-scheduler-bench/src/main.rs
index 2fd571aaa18882..1b2025a5fc7d44 100644
--- a/transaction-scheduler-bench/src/main.rs
+++ b/transaction-scheduler-bench/src/main.rs
@@ -82,7 +82,7 @@ type PreprocessedTransaction = (SanitizedTransaction, Vec<solana_scheduler::Lock
 type TransactionMessage = Box<PreprocessedTransaction>;
 type CompletedTransactionMessage = (usize, TransactionMessage); // thread index and transaction message
 type TransactionBatchMessage = Vec<TransactionMessage>;
-type BatchSenderMessage = Vec<Vec<PreprocessedTransaction>>;
+type BatchSenderMessage = solana_scheduler::MultiplexedPayload; // Vec<Vec<PreprocessedTransaction>>;
 
 #[derive(Debug, Default)]
 struct TransactionSchedulerBenchMetrics {

From ee93505ed79d395f0c5721595bf44c77a911e67f Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 5 Aug 2022 22:53:14 +0900
Subject: [PATCH 0049/3199] save

---
 transaction-scheduler-bench/src/main.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/transaction-scheduler-bench/src/main.rs b/transaction-scheduler-bench/src/main.rs
index 1b2025a5fc7d44..7e72e99b201846 100644
--- a/transaction-scheduler-bench/src/main.rs
+++ b/transaction-scheduler-bench/src/main.rs
@@ -117,7 +117,7 @@ struct PacketSendingConfig {
 }
 
 fn spawn_unified_scheduler(
-        packet_batch_receiver: Receiver<Vec<Vec<PreprocessedTransaction>>>,
+        packet_batch_receiver: Receiver<BatchSenderMessage>,
         transaction_batch_senders: Vec<Sender<TransactionBatchMessage>>,
         completed_transaction_receiver: Receiver<CompletedTransactionMessage>,
         bank_forks: Arc<RwLock<BankForks>>,

From d5cc67ada3eb4feb668e75337c8a3d57bafbf863 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 5 Aug 2022 22:53:32 +0900
Subject: [PATCH 0050/3199] save

---
 transaction-scheduler-bench/src/main.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/transaction-scheduler-bench/src/main.rs b/transaction-scheduler-bench/src/main.rs
index 7e72e99b201846..a9274429c46be8 100644
--- a/transaction-scheduler-bench/src/main.rs
+++ b/transaction-scheduler-bench/src/main.rs
@@ -382,7 +382,7 @@ fn spawn_packet_sender(
     metrics: Arc<TransactionSchedulerBenchMetrics>,
     num_accounts: usize,
     accounts: Arc<Vec<Keypair>>,
-    packet_batch_sender: Sender<Vec<Vec<PreprocessedTransaction>>>,
+    packet_batch_sender: Sender<BatchSenderMessage>,
     config: Arc<PacketSendingConfig>,
     duration: Duration,
     exit: Arc<AtomicBool>,

From 743dcafb22c690cbf335097172efbeb797e2084a Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 5 Aug 2022 22:53:50 +0900
Subject: [PATCH 0051/3199] save

---
 transaction-scheduler-bench/src/main.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/transaction-scheduler-bench/src/main.rs b/transaction-scheduler-bench/src/main.rs
index a9274429c46be8..65464f4d55a8f4 100644
--- a/transaction-scheduler-bench/src/main.rs
+++ b/transaction-scheduler-bench/src/main.rs
@@ -353,7 +353,7 @@ fn spawn_packet_senders(
     metrics: Arc<TransactionSchedulerBenchMetrics>,
     high_conflict_sender: usize,
     accounts: Arc<Vec<Keypair>>,
-    packet_batch_sender: Sender<Vec<Vec<PreprocessedTransaction>>>,
+    packet_batch_sender: Sender<BatchSenderMessage>,
     config: Arc<PacketSendingConfig>,
     duration: Duration,
     exit: Arc<AtomicBool>,

From 2a8f3abcabb44fda50552e4723e9ed7812eb1dca Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 5 Aug 2022 22:54:22 +0900
Subject: [PATCH 0052/3199] save

---
 scheduler/src/lib.rs | 1 +
 1 file changed, 1 insertion(+)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index b3cca1d2231d50..e6e04a1508fe9a 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -327,6 +327,7 @@ fn attempt_lock_for_execution<'a>(
 // switched from crossbeam_channel::select! due to observed poor performance
 pub enum MultiplexedPayload {
     FromPrevious((Weight, Box<(SanitizedTransaction, Vec<LockAttempt>)>)),
+    FromPreviousBatched(()),
     FromExecute(Box<ExecutionEnvironment>),
 }
 

From 21d1a77ca06d6f9612cbefd2f8385d00ced8f71c Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 5 Aug 2022 22:54:42 +0900
Subject: [PATCH 0053/3199] save

---
 scheduler/src/lib.rs | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index e6e04a1508fe9a..f44ea97c6b817d 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -603,6 +603,11 @@ impl ScheduleStage {
 
                     Self::register_runnable_task(weighted_tx, runnable_queue);
                 }
+                MultiplexedPayload::FromPreviousBatched(weighted_txes) => {
+                    trace!("recv from previous");
+
+                    Self::register_runnable_task(weighted_tx, runnable_queue);
+                }
                 MultiplexedPayload::FromExecute(mut processed_execution_environment) => {
                     trace!("recv from execute");
                     executing_queue_count -= 1;

From 5344f423699dbb26aef91e66ec690202f539da5d Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 5 Aug 2022 22:55:02 +0900
Subject: [PATCH 0054/3199] save

---
 scheduler/src/lib.rs | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index f44ea97c6b817d..f30a058e08c527 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -606,7 +606,8 @@ impl ScheduleStage {
                 MultiplexedPayload::FromPreviousBatched(weighted_txes) => {
                     trace!("recv from previous");
 
-                    Self::register_runnable_task(weighted_tx, runnable_queue);
+                    //Self::register_runnable_task(weighted_tx, runnable_queue);
+                    panic!()
                 }
                 MultiplexedPayload::FromExecute(mut processed_execution_environment) => {
                     trace!("recv from execute");

From edccb9295749339cab5d129c2e398165be4d2322 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 5 Aug 2022 22:56:13 +0900
Subject: [PATCH 0055/3199] save

---
 transaction-scheduler-bench/src/main.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/transaction-scheduler-bench/src/main.rs b/transaction-scheduler-bench/src/main.rs
index 65464f4d55a8f4..17449f5290593e 100644
--- a/transaction-scheduler-bench/src/main.rs
+++ b/transaction-scheduler-bench/src/main.rs
@@ -78,7 +78,7 @@ struct Args {
 }
 
 /// Some convenient type aliases
-type PreprocessedTransaction = (SanitizedTransaction, Vec<solana_scheduler::LockAttempt>);
+type PreprocessedTransaction = Box<(SanitizedTransaction, Vec<solana_scheduler::LockAttempt>)>;
 type TransactionMessage = Box<PreprocessedTransaction>;
 type CompletedTransactionMessage = (usize, TransactionMessage); // thread index and transaction message
 type TransactionBatchMessage = Vec<TransactionMessage>;

From a560a45f7fb13ab89cd2510a481899d9a8a3a003 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 5 Aug 2022 22:56:36 +0900
Subject: [PATCH 0056/3199] save

---
 transaction-scheduler-bench/src/main.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/transaction-scheduler-bench/src/main.rs b/transaction-scheduler-bench/src/main.rs
index 17449f5290593e..b7c057b36d2aaf 100644
--- a/transaction-scheduler-bench/src/main.rs
+++ b/transaction-scheduler-bench/src/main.rs
@@ -513,7 +513,7 @@ fn build_packet(
         .map(|address| solana_scheduler::LockAttempt::new(**address, solana_scheduler::RequestedUsage::Readonly));
     let locks = writable_lock_iter.chain(readonly_lock_iter).collect::<Vec<_>>();
 
-    (sanitized_tx, locks)
+    Box::new((sanitized_tx, locks))
 }
 
 fn build_accounts(num_accounts: usize) -> Vec<Keypair> {

From 49ee544d67e339c0fd6dbe5f772465d6ff4b562b Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 5 Aug 2022 22:58:01 +0900
Subject: [PATCH 0057/3199] save

---
 scheduler/src/lib.rs | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index f30a058e08c527..fdf241f52b997c 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -322,12 +322,13 @@ fn attempt_lock_for_execution<'a>(
     (all_succeeded_so_far, placeholder_attempts)
 }
 
+type PreprocessedTransaction = (SanitizedTransaction, Vec<LockAttempt>);
 // multiplexed to reduce the futex syscal per tx down to minimum and to make the schduler to
 // adaptive relative load between sigverify stage and execution substage
 // switched from crossbeam_channel::select! due to observed poor performance
 pub enum MultiplexedPayload {
-    FromPrevious((Weight, Box<(SanitizedTransaction, Vec<LockAttempt>)>)),
-    FromPreviousBatched(()),
+    FromPrevious((Weight, Box<PreprocessedTransaction>)),
+    FromPreviousBatched(Vec<Vec<Box<PreprocessedTransaction>>>),
     FromExecute(Box<ExecutionEnvironment>),
 }
 

From 0b8a8faa2ddc1e78639d71d11ab30813b971d45d Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 5 Aug 2022 23:00:54 +0900
Subject: [PATCH 0058/3199] save

---
 scheduler/src/lib.rs | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index fdf241f52b997c..f0c02ca9df8b53 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -608,7 +608,12 @@ impl ScheduleStage {
                     trace!("recv from previous");
 
                     //Self::register_runnable_task(weighted_tx, runnable_queue);
-                    panic!()
+                    for vv in vv {
+                        for v in vv {
+                            let p = v.0.get_transaction_priority_details().unwrap().priority;
+                            Self::register_runnable_task((Weight { weight: p }, p));
+                        }
+                    }
                 }
                 MultiplexedPayload::FromExecute(mut processed_execution_environment) => {
                     trace!("recv from execute");

From a683bb393fd916a8e883213f937b1f303ec5f94a Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 5 Aug 2022 23:01:05 +0900
Subject: [PATCH 0059/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index f0c02ca9df8b53..0eda1b9b8d425e 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -611,7 +611,7 @@ impl ScheduleStage {
                     for vv in vv {
                         for v in vv {
                             let p = v.0.get_transaction_priority_details().unwrap().priority;
-                            Self::register_runnable_task((Weight { weight: p }, p));
+                            Self::register_runnable_task((Weight { ix: p }, p));
                         }
                     }
                 }

From a473ee8062c9b5adc4ade1e55263b93e1ad6e1f3 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 5 Aug 2022 23:01:16 +0900
Subject: [PATCH 0060/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 0eda1b9b8d425e..24050234cabce7 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -611,7 +611,7 @@ impl ScheduleStage {
                     for vv in vv {
                         for v in vv {
                             let p = v.0.get_transaction_priority_details().unwrap().priority;
-                            Self::register_runnable_task((Weight { ix: p }, p));
+                            Self::register_runnable_task((Weight { ix: p }, p), runnable_queue);
                         }
                     }
                 }

From f9f1bc53b566412e581d4ab54ad43a95e3d0e5af Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 5 Aug 2022 23:01:27 +0900
Subject: [PATCH 0061/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 24050234cabce7..733e97149cc6b0 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -611,7 +611,7 @@ impl ScheduleStage {
                     for vv in vv {
                         for v in vv {
                             let p = v.0.get_transaction_priority_details().unwrap().priority;
-                            Self::register_runnable_task((Weight { ix: p }, p), runnable_queue);
+                            Self::register_runnable_task((Weight { ix: p }, v), runnable_queue);
                         }
                     }
                 }

From ca52e96d6aa8c43dad2214f27fc7742e2aafe5f5 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 5 Aug 2022 23:01:43 +0900
Subject: [PATCH 0062/3199] save

---
 scheduler/src/lib.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 733e97149cc6b0..8a0c75a3cfc0ba 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -604,11 +604,11 @@ impl ScheduleStage {
 
                     Self::register_runnable_task(weighted_tx, runnable_queue);
                 }
-                MultiplexedPayload::FromPreviousBatched(weighted_txes) => {
+                MultiplexedPayload::FromPreviousBatched(vvv) => {
                     trace!("recv from previous");
 
                     //Self::register_runnable_task(weighted_tx, runnable_queue);
-                    for vv in vv {
+                    for vv in vvv {
                         for v in vv {
                             let p = v.0.get_transaction_priority_details().unwrap().priority;
                             Self::register_runnable_task((Weight { ix: p }, v), runnable_queue);

From 3947376d19fe9b6d1b4f967caaf3d76175bdb6af Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 5 Aug 2022 23:06:06 +0900
Subject: [PATCH 0063/3199] save

---
 scheduler/src/lib.rs | 15 ++++++++++++++-
 1 file changed, 14 insertions(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 8a0c75a3cfc0ba..d260d0fcc9f20f 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -332,6 +332,19 @@ pub enum MultiplexedPayload {
     FromExecute(Box<ExecutionEnvironment>),
 }
 
+fn get_transaction_priority_details(tx: &SanitizedTransaction) -> u64 {
+        let mut compute_budget = ComputeBudget::default();
+        let prioritization_fee_details = compute_budget
+            .process_instructions(
+                tx.message().program_instructions_iter(),
+                true, // use default units per instruction
+                true, // don't reject txs that use set compute unit price ix
+            )
+            .ok()?;
+
+            prioritization_fee_details.get_priority()
+}
+
 pub struct ScheduleStage {}
 
 impl ScheduleStage {
@@ -610,7 +623,7 @@ impl ScheduleStage {
                     //Self::register_runnable_task(weighted_tx, runnable_queue);
                     for vv in vvv {
                         for v in vv {
-                            let p = v.0.get_transaction_priority_details().unwrap().priority;
+                            let p = get_transaction_priority_details(v.0).unwrap().priority;
                             Self::register_runnable_task((Weight { ix: p }, v), runnable_queue);
                         }
                     }

From 211c585dd33c083001f72389eb5ef613a3821b67 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 5 Aug 2022 23:06:14 +0900
Subject: [PATCH 0064/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index d260d0fcc9f20f..99683084e15af6 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -623,7 +623,7 @@ impl ScheduleStage {
                     //Self::register_runnable_task(weighted_tx, runnable_queue);
                     for vv in vvv {
                         for v in vv {
-                            let p = get_transaction_priority_details(v.0).unwrap().priority;
+                            let p = get_transaction_priority_details(&v.0).unwrap().priority;
                             Self::register_runnable_task((Weight { ix: p }, v), runnable_queue);
                         }
                     }

From 73d3ae4417d4110bffb9fe36e514387fb0522a88 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 5 Aug 2022 23:06:24 +0900
Subject: [PATCH 0065/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 99683084e15af6..57b90253d2e862 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -623,7 +623,7 @@ impl ScheduleStage {
                     //Self::register_runnable_task(weighted_tx, runnable_queue);
                     for vv in vvv {
                         for v in vv {
-                            let p = get_transaction_priority_details(&v.0).unwrap().priority;
+                            let p = get_transaction_priority_details(&v.0);
                             Self::register_runnable_task((Weight { ix: p }, v), runnable_queue);
                         }
                     }

From 8c4386bcaf312c6c596f8ffe5b438cfcf18aee3b Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 5 Aug 2022 23:07:39 +0900
Subject: [PATCH 0066/3199] save

---
 scheduler/src/lib.rs | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 57b90253d2e862..16f7e43a603b61 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -340,9 +340,7 @@ fn get_transaction_priority_details(tx: &SanitizedTransaction) -> u64 {
                 true, // use default units per instruction
                 true, // don't reject txs that use set compute unit price ix
             )
-            .ok()?;
-
-            prioritization_fee_details.get_priority()
+            .map(|d| d.get_priority()).unwrap_or_default()
 }
 
 pub struct ScheduleStage {}

From 1c29eec0ea2ad709c34fdcfab21b32f89e060bd7 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 5 Aug 2022 23:08:03 +0900
Subject: [PATCH 0067/3199] save

---
 scheduler/src/lib.rs | 1 +
 1 file changed, 1 insertion(+)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 16f7e43a603b61..7ab4827aabd839 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -333,6 +333,7 @@ pub enum MultiplexedPayload {
 }
 
 fn get_transaction_priority_details(tx: &SanitizedTransaction) -> u64 {
+        use solana_program_runtime::compute_budget::ComputeBudget;
         let mut compute_budget = ComputeBudget::default();
         let prioritization_fee_details = compute_budget
             .process_instructions(

From acc11e07c4bd426872e367e5836a6ea439f8ad5c Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 5 Aug 2022 23:08:19 +0900
Subject: [PATCH 0068/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 7ab4827aabd839..569bb7db08b7f0 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -335,7 +335,7 @@ pub enum MultiplexedPayload {
 fn get_transaction_priority_details(tx: &SanitizedTransaction) -> u64 {
         use solana_program_runtime::compute_budget::ComputeBudget;
         let mut compute_budget = ComputeBudget::default();
-        let prioritization_fee_details = compute_budget
+        compute_budget
             .process_instructions(
                 tx.message().program_instructions_iter(),
                 true, // use default units per instruction

From 3c3cdfa65d20fd649abcbee699f8a78bacfbd8ae Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 5 Aug 2022 23:08:42 +0900
Subject: [PATCH 0069/3199] save

---
 scheduler/Cargo.toml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/scheduler/Cargo.toml b/scheduler/Cargo.toml
index ae9f9b292419b6..308dcef5aaf11a 100644
--- a/scheduler/Cargo.toml
+++ b/scheduler/Cargo.toml
@@ -16,6 +16,7 @@ crossbeam-channel = "0.5.5"
 log = "0.4.17"
 solana-entry = { path = "../entry", version = "=1.11.5" }
 solana-runtime = { path = "../runtime", version = "=1.11.5" }
+solana-program-runtime = { path = "../program-runtime", version = "=1.11.5" }
 solana-sdk = { path = "../sdk", version = "=1.11.5" }
 solana-logger = { path = "../logger", version = "=1.11.5" }
 solana-version = { path = "../version", version = "=1.11.5" }

From d34a4c4242aa431c7a73e87274064bdd9c0d713d Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 5 Aug 2022 23:09:02 +0900
Subject: [PATCH 0070/3199] save

---
 Cargo.lock           | 1 +
 scheduler/src/lib.rs | 2 +-
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/Cargo.lock b/Cargo.lock
index 2e35573b8daa73..46428aa41fe49a 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -6075,6 +6075,7 @@ dependencies = [
  "solana-logger 1.11.5",
  "solana-measure",
  "solana-metrics",
+ "solana-program-runtime",
  "solana-runtime",
  "solana-sdk 1.11.5",
  "solana-version",
diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 569bb7db08b7f0..b0fa08591e2d27 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -238,7 +238,7 @@ impl AddressBook {
 #[derive(Copy, Clone, Debug, PartialEq, Eq, PartialOrd, Ord)]
 pub struct Weight {
     // naming: Sequence Ordering?
-    pub ix: usize, // index in ledger entry?
+    pub ix: u64, // index in ledger entry?
                    // gas fee
 }
 

From afe6030e8515970492f677a6bc16a07ea2b6363a Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 5 Aug 2022 23:09:54 +0900
Subject: [PATCH 0071/3199] save

---
 scheduler/src/lib.rs | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index b0fa08591e2d27..fe0bcc550d0586 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -248,7 +248,7 @@ pub struct UniqueWeight {
     weight: Weight,
     // we can't use Transaction::message_hash because it's manipulatable to be favorous to the tx
     // submitter
-    //unique_key: Hash, // tie breaker? random noise? also for unique identification of txes?
+    unique_key: Hash, // tie breaker? random noise? also for unique identification of txes?
     // fee?
 }
 /*
@@ -352,7 +352,7 @@ impl ScheduleStage {
         runnable_queue: &mut TaskQueue,
     ) {
         // manage randomness properly for future scheduling determinism
-        //let mut rng = rand::thread_rng();
+        let mut rng = rand::thread_rng();
 
         //let ix = 23;
         //let tx = bank
@@ -366,7 +366,7 @@ impl ScheduleStage {
         runnable_queue.add_to_schedule(
             UniqueWeight {
                 weight,
-                //unique_key: solana_sdk::hash::new_rand(&mut rng),
+                unique_key: solana_sdk::hash::new_rand(&mut rng),
             },
             Task { tx },
         );

From 545817583aaa6cd11d5635926ee85d2d787b1b0f Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 5 Aug 2022 23:11:50 +0900
Subject: [PATCH 0072/3199] save

---
 transaction-scheduler-bench/src/main.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/transaction-scheduler-bench/src/main.rs b/transaction-scheduler-bench/src/main.rs
index b7c057b36d2aaf..1a74f225148950 100644
--- a/transaction-scheduler-bench/src/main.rs
+++ b/transaction-scheduler-bench/src/main.rs
@@ -135,7 +135,7 @@ fn spawn_unified_scheduler(
             &mut contended_queue,
             &mut address_book,
             &packet_batch_receiver,
-            3, //&pre_execute_env_sender,
+            &transaction_batch_senders[0]
             4, //&post_schedule_env_sender,
         );
     })

From da7ec1abf00956a48cc22d1938c255b045aeb5a0 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 5 Aug 2022 23:11:55 +0900
Subject: [PATCH 0073/3199] save

---
 transaction-scheduler-bench/src/main.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/transaction-scheduler-bench/src/main.rs b/transaction-scheduler-bench/src/main.rs
index 1a74f225148950..f6fc183ccaa15e 100644
--- a/transaction-scheduler-bench/src/main.rs
+++ b/transaction-scheduler-bench/src/main.rs
@@ -135,7 +135,7 @@ fn spawn_unified_scheduler(
             &mut contended_queue,
             &mut address_book,
             &packet_batch_receiver,
-            &transaction_batch_senders[0]
+            &transaction_batch_senders[0],
             4, //&post_schedule_env_sender,
         );
     })

From 3f271b5f7ecf46c382e65d88a5bbb83d2ba726d8 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 5 Aug 2022 23:13:30 +0900
Subject: [PATCH 0074/3199] save

---
 transaction-scheduler-bench/src/main.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/transaction-scheduler-bench/src/main.rs b/transaction-scheduler-bench/src/main.rs
index f6fc183ccaa15e..47243fffcea49e 100644
--- a/transaction-scheduler-bench/src/main.rs
+++ b/transaction-scheduler-bench/src/main.rs
@@ -79,7 +79,7 @@ struct Args {
 
 /// Some convenient type aliases
 type PreprocessedTransaction = Box<(SanitizedTransaction, Vec<solana_scheduler::LockAttempt>)>;
-type TransactionMessage = Box<PreprocessedTransaction>;
+type TransactionMessage = PreprocessedTransaction;
 type CompletedTransactionMessage = (usize, TransactionMessage); // thread index and transaction message
 type TransactionBatchMessage = Vec<TransactionMessage>;
 type BatchSenderMessage = solana_scheduler::MultiplexedPayload; // Vec<Vec<PreprocessedTransaction>>;

From 5962ec64107eb07d97a5cc896197c97230f12fb5 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 5 Aug 2022 23:14:21 +0900
Subject: [PATCH 0075/3199] save

---
 transaction-scheduler-bench/src/main.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/transaction-scheduler-bench/src/main.rs b/transaction-scheduler-bench/src/main.rs
index 47243fffcea49e..0d1f304fe2e045 100644
--- a/transaction-scheduler-bench/src/main.rs
+++ b/transaction-scheduler-bench/src/main.rs
@@ -81,7 +81,7 @@ struct Args {
 type PreprocessedTransaction = Box<(SanitizedTransaction, Vec<solana_scheduler::LockAttempt>)>;
 type TransactionMessage = PreprocessedTransaction;
 type CompletedTransactionMessage = (usize, TransactionMessage); // thread index and transaction message
-type TransactionBatchMessage = Vec<TransactionMessage>;
+type TransactionBatchMessage = Box<solana_scheduler::ExecutionEnvironment>// Vec<TransactionMessage>;
 type BatchSenderMessage = solana_scheduler::MultiplexedPayload; // Vec<Vec<PreprocessedTransaction>>;
 
 #[derive(Debug, Default)]

From af090e7fe164b43069648dc252bdec155b29f163 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 5 Aug 2022 23:14:27 +0900
Subject: [PATCH 0076/3199] save

---
 transaction-scheduler-bench/src/main.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/transaction-scheduler-bench/src/main.rs b/transaction-scheduler-bench/src/main.rs
index 0d1f304fe2e045..ff1f4e7c051252 100644
--- a/transaction-scheduler-bench/src/main.rs
+++ b/transaction-scheduler-bench/src/main.rs
@@ -81,7 +81,7 @@ struct Args {
 type PreprocessedTransaction = Box<(SanitizedTransaction, Vec<solana_scheduler::LockAttempt>)>;
 type TransactionMessage = PreprocessedTransaction;
 type CompletedTransactionMessage = (usize, TransactionMessage); // thread index and transaction message
-type TransactionBatchMessage = Box<solana_scheduler::ExecutionEnvironment>// Vec<TransactionMessage>;
+type TransactionBatchMessage = Box<solana_scheduler::ExecutionEnvironment>; // Vec<TransactionMessage>;
 type BatchSenderMessage = solana_scheduler::MultiplexedPayload; // Vec<Vec<PreprocessedTransaction>>;
 
 #[derive(Debug, Default)]

From dc7e302cfabc08616ca28eb1f77099b1ab765b98 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 5 Aug 2022 23:15:33 +0900
Subject: [PATCH 0077/3199] save

---
 transaction-scheduler-bench/src/main.rs | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/transaction-scheduler-bench/src/main.rs b/transaction-scheduler-bench/src/main.rs
index ff1f4e7c051252..f3273e344a6bd4 100644
--- a/transaction-scheduler-bench/src/main.rs
+++ b/transaction-scheduler-bench/src/main.rs
@@ -328,10 +328,7 @@ fn handle_transaction_batch(
         num_transactions * execution_per_tx_us,
     ));
 
-    let priority_collected = transaction_batch
-        .iter()
-        .map(|tx| tx.0.get_transaction_priority_details().unwrap().priority)
-        .sum();
+    let priority_collected = transaction_batch.tx.get_transaction_priority_details().unwrap().priority;
 
     metrics
         .num_transactions_completed

From 6a2abd821af68c181a72de5756199e81ad0d9a88 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 5 Aug 2022 23:16:31 +0900
Subject: [PATCH 0078/3199] save

---
 transaction-scheduler-bench/src/main.rs | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/transaction-scheduler-bench/src/main.rs b/transaction-scheduler-bench/src/main.rs
index f3273e344a6bd4..d78bc7e3578a38 100644
--- a/transaction-scheduler-bench/src/main.rs
+++ b/transaction-scheduler-bench/src/main.rs
@@ -337,11 +337,9 @@ fn handle_transaction_batch(
         .priority_collected
         .fetch_add(priority_collected, Ordering::Relaxed);
 
-    for transaction in transaction_batch {
         completed_transaction_sender
-            .send((thread_index, transaction))
+            .send((thread_index, transaction_batch.tx))
             .unwrap();
-    }
 }
 
 const NUM_SENDERS: usize = 4;

From ca981e6b9b2c5ed25add8b9428fa11f750d80a25 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 5 Aug 2022 23:17:00 +0900
Subject: [PATCH 0079/3199] save

---
 transaction-scheduler-bench/src/main.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/transaction-scheduler-bench/src/main.rs b/transaction-scheduler-bench/src/main.rs
index d78bc7e3578a38..a227550a7d94c2 100644
--- a/transaction-scheduler-bench/src/main.rs
+++ b/transaction-scheduler-bench/src/main.rs
@@ -328,7 +328,7 @@ fn handle_transaction_batch(
         num_transactions * execution_per_tx_us,
     ));
 
-    let priority_collected = transaction_batch.tx.get_transaction_priority_details().unwrap().priority;
+    let priority_collected = transaction_batch.task.tx.get_transaction_priority_details().unwrap().priority;
 
     metrics
         .num_transactions_completed
@@ -338,7 +338,7 @@ fn handle_transaction_batch(
         .fetch_add(priority_collected, Ordering::Relaxed);
 
         completed_transaction_sender
-            .send((thread_index, transaction_batch.tx))
+            .send((thread_index, transaction_batch))
             .unwrap();
 }
 

From 949793a43ff607b719fd393a91052412a2b42453 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 5 Aug 2022 23:17:19 +0900
Subject: [PATCH 0080/3199] save

---
 transaction-scheduler-bench/src/main.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/transaction-scheduler-bench/src/main.rs b/transaction-scheduler-bench/src/main.rs
index a227550a7d94c2..7c25865b394a31 100644
--- a/transaction-scheduler-bench/src/main.rs
+++ b/transaction-scheduler-bench/src/main.rs
@@ -319,7 +319,7 @@ fn handle_transaction_batch(
     transaction_batch: TransactionBatchMessage,
     execution_per_tx_us: u64,
 ) {
-    let num_transactions = transaction_batch.len() as u64;
+    let num_transactions = 1; //transaction_batch.len() as u64;
     metrics
         .num_transactions_scheduled
         .fetch_add(num_transactions as usize, Ordering::Relaxed);

From 45cf314e431304721e5bf23294577014db0ef643 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 5 Aug 2022 23:17:34 +0900
Subject: [PATCH 0081/3199] save

---
 transaction-scheduler-bench/src/main.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/transaction-scheduler-bench/src/main.rs b/transaction-scheduler-bench/src/main.rs
index 7c25865b394a31..3d8626ddae45fc 100644
--- a/transaction-scheduler-bench/src/main.rs
+++ b/transaction-scheduler-bench/src/main.rs
@@ -328,7 +328,7 @@ fn handle_transaction_batch(
         num_transactions * execution_per_tx_us,
     ));
 
-    let priority_collected = transaction_batch.task.tx.get_transaction_priority_details().unwrap().priority;
+    let priority_collected = transaction_batch.task.tx.0.get_transaction_priority_details().unwrap().priority;
 
     metrics
         .num_transactions_completed

From 105a396fe25e5fd6cd4a63e233cd696cef1dbe33 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 5 Aug 2022 23:18:09 +0900
Subject: [PATCH 0082/3199] save

---
 transaction-scheduler-bench/src/main.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/transaction-scheduler-bench/src/main.rs b/transaction-scheduler-bench/src/main.rs
index 3d8626ddae45fc..d4b70cf480de65 100644
--- a/transaction-scheduler-bench/src/main.rs
+++ b/transaction-scheduler-bench/src/main.rs
@@ -80,7 +80,7 @@ struct Args {
 /// Some convenient type aliases
 type PreprocessedTransaction = Box<(SanitizedTransaction, Vec<solana_scheduler::LockAttempt>)>;
 type TransactionMessage = PreprocessedTransaction;
-type CompletedTransactionMessage = (usize, TransactionMessage); // thread index and transaction message
+type CompletedTransactionMessage = (usize, solana_scheduler::ExecutionEnvironment); //(usize, TransactionMessage); // thread index and transaction message
 type TransactionBatchMessage = Box<solana_scheduler::ExecutionEnvironment>; // Vec<TransactionMessage>;
 type BatchSenderMessage = solana_scheduler::MultiplexedPayload; // Vec<Vec<PreprocessedTransaction>>;
 

From cfb4c330ea20819bd56feb91ea4b377f9713ec3f Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 5 Aug 2022 23:18:31 +0900
Subject: [PATCH 0083/3199] save

---
 transaction-scheduler-bench/src/main.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/transaction-scheduler-bench/src/main.rs b/transaction-scheduler-bench/src/main.rs
index d4b70cf480de65..52cd11da58edc3 100644
--- a/transaction-scheduler-bench/src/main.rs
+++ b/transaction-scheduler-bench/src/main.rs
@@ -80,7 +80,7 @@ struct Args {
 /// Some convenient type aliases
 type PreprocessedTransaction = Box<(SanitizedTransaction, Vec<solana_scheduler::LockAttempt>)>;
 type TransactionMessage = PreprocessedTransaction;
-type CompletedTransactionMessage = (usize, solana_scheduler::ExecutionEnvironment); //(usize, TransactionMessage); // thread index and transaction message
+type CompletedTransactionMessage = (usize, Box<solana_scheduler::ExecutionEnvironment>); //(usize, TransactionMessage); // thread index and transaction message
 type TransactionBatchMessage = Box<solana_scheduler::ExecutionEnvironment>; // Vec<TransactionMessage>;
 type BatchSenderMessage = solana_scheduler::MultiplexedPayload; // Vec<Vec<PreprocessedTransaction>>;
 

From da1d029430b080668b7ec8110590737f0f812f2d Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 5 Aug 2022 23:21:49 +0900
Subject: [PATCH 0084/3199] save

---
 scheduler/src/lib.rs                    | 4 ++--
 transaction-scheduler-bench/src/main.rs | 4 ++--
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index fe0bcc550d0586..a7abec2a022b90 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -602,7 +602,7 @@ impl ScheduleStage {
         address_book: &mut AddressBook,
         from: &crossbeam_channel::Receiver<MultiplexedPayload>,
         to_execute_substage: &crossbeam_channel::Sender<Box<ExecutionEnvironment>>,
-        to_next_stage: &crossbeam_channel::Sender<Box<ExecutionEnvironment>>, // assume nonblocking
+        // to_next_stage: &crossbeam_channel::Sender<Box<ExecutionEnvironment>>, // assume nonblocking
     ) {
         let mut executing_queue_count = 0;
 
@@ -633,7 +633,7 @@ impl ScheduleStage {
 
                     Self::commit_result(&mut processed_execution_environment, address_book);
                     // async-ly propagate the result to rpc subsystems
-                    to_next_stage.send(processed_execution_environment).unwrap();
+                    // to_next_stage.send(processed_execution_environment).unwrap();
                 }
             }
 
diff --git a/transaction-scheduler-bench/src/main.rs b/transaction-scheduler-bench/src/main.rs
index 52cd11da58edc3..313545cfb1f419 100644
--- a/transaction-scheduler-bench/src/main.rs
+++ b/transaction-scheduler-bench/src/main.rs
@@ -119,7 +119,7 @@ struct PacketSendingConfig {
 fn spawn_unified_scheduler(
         packet_batch_receiver: Receiver<BatchSenderMessage>,
         transaction_batch_senders: Vec<Sender<TransactionBatchMessage>>,
-        completed_transaction_receiver: Receiver<CompletedTransactionMessage>,
+        //completed_transaction_receiver: Receiver<CompletedTransactionMessage>,
         bank_forks: Arc<RwLock<BankForks>>,
         max_batch_size: usize,
         exit: Arc<AtomicBool>,
@@ -136,7 +136,7 @@ fn spawn_unified_scheduler(
             &mut address_book,
             &packet_batch_receiver,
             &transaction_batch_senders[0],
-            4, //&post_schedule_env_sender,
+            //&completed_transaction_receiver
         );
     })
 }

From c2cab61d2eb80c332f8141f848c21ba1f21a3f42 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 5 Aug 2022 23:23:03 +0900
Subject: [PATCH 0085/3199] save

---
 transaction-scheduler-bench/src/main.rs | 7 ++-----
 1 file changed, 2 insertions(+), 5 deletions(-)

diff --git a/transaction-scheduler-bench/src/main.rs b/transaction-scheduler-bench/src/main.rs
index 313545cfb1f419..8c265e29f4be4d 100644
--- a/transaction-scheduler-bench/src/main.rs
+++ b/transaction-scheduler-bench/src/main.rs
@@ -80,7 +80,7 @@ struct Args {
 /// Some convenient type aliases
 type PreprocessedTransaction = Box<(SanitizedTransaction, Vec<solana_scheduler::LockAttempt>)>;
 type TransactionMessage = PreprocessedTransaction;
-type CompletedTransactionMessage = (usize, Box<solana_scheduler::ExecutionEnvironment>); //(usize, TransactionMessage); // thread index and transaction message
+type CompletedTransactionMessage = solana_scheduler::MultiplexedPayload; // (usize, Box<solana_scheduler::ExecutionEnvironment>); //(usize, TransactionMessage); // thread index and transaction message
 type TransactionBatchMessage = Box<solana_scheduler::ExecutionEnvironment>; // Vec<TransactionMessage>;
 type BatchSenderMessage = solana_scheduler::MultiplexedPayload; // Vec<Vec<PreprocessedTransaction>>;
 
@@ -163,8 +163,6 @@ fn main() {
     let (packet_batch_sender, packet_batch_receiver) = crossbeam_channel::unbounded();
     let (transaction_batch_senders, transaction_batch_receivers) =
         build_channels(num_execution_threads);
-    let (completed_transaction_sender, completed_transaction_receiver) =
-        crossbeam_channel::unbounded();
     let bank_forks = Arc::new(RwLock::new(BankForks::new(Bank::default_for_tests())));
     let exit = Arc::new(AtomicBool::new(false));
 
@@ -172,7 +170,6 @@ fn main() {
     let scheduler_handle = spawn_unified_scheduler(
         packet_batch_receiver,
         transaction_batch_senders,
-        completed_transaction_receiver,
         bank_forks,
         max_batch_size,
         exit.clone(),
@@ -184,7 +181,7 @@ fn main() {
     let execution_handles = start_execution_threads(
         metrics.clone(),
         transaction_batch_receivers,
-        completed_transaction_sender,
+        packet_batch_sender,
         execution_per_tx_us,
         exit.clone(),
     );

From 0c83d0e431755d976530fe6c1784a592663f7779 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 5 Aug 2022 23:23:46 +0900
Subject: [PATCH 0086/3199] save

---
 transaction-scheduler-bench/src/main.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/transaction-scheduler-bench/src/main.rs b/transaction-scheduler-bench/src/main.rs
index 8c265e29f4be4d..e82e9445f0978a 100644
--- a/transaction-scheduler-bench/src/main.rs
+++ b/transaction-scheduler-bench/src/main.rs
@@ -335,7 +335,7 @@ fn handle_transaction_batch(
         .fetch_add(priority_collected, Ordering::Relaxed);
 
         completed_transaction_sender
-            .send((thread_index, transaction_batch))
+            .send(solana_scheduler::MultiplexedPayload::FromExecute(transaction_batch))
             .unwrap();
 }
 

From 9eb28cf152158c13ca8aef5abc2a1157dd4ead67 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 5 Aug 2022 23:24:01 +0900
Subject: [PATCH 0087/3199] save

---
 transaction-scheduler-bench/src/main.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/transaction-scheduler-bench/src/main.rs b/transaction-scheduler-bench/src/main.rs
index e82e9445f0978a..2d6e0021437ece 100644
--- a/transaction-scheduler-bench/src/main.rs
+++ b/transaction-scheduler-bench/src/main.rs
@@ -181,7 +181,7 @@ fn main() {
     let execution_handles = start_execution_threads(
         metrics.clone(),
         transaction_batch_receivers,
-        packet_batch_sender,
+        packet_batch_sender.clone(),
         execution_per_tx_us,
         exit.clone(),
     );

From 7135ea8ff17f241844c6bd187866fc52b898c877 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 5 Aug 2022 23:24:19 +0900
Subject: [PATCH 0088/3199] save

---
 transaction-scheduler-bench/src/main.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/transaction-scheduler-bench/src/main.rs b/transaction-scheduler-bench/src/main.rs
index 2d6e0021437ece..c7fd0d36fd2503 100644
--- a/transaction-scheduler-bench/src/main.rs
+++ b/transaction-scheduler-bench/src/main.rs
@@ -134,7 +134,7 @@ fn spawn_unified_scheduler(
             &mut runnable_queue,
             &mut contended_queue,
             &mut address_book,
-            &packet_batch_receiver,
+            packet_batch_receiver.clone(),
             &transaction_batch_senders[0],
             //&completed_transaction_receiver
         );

From b591aafac6cc38d4efbdf1da5c9f4ad79b61b563 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 5 Aug 2022 23:24:27 +0900
Subject: [PATCH 0089/3199] save

---
 transaction-scheduler-bench/src/main.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/transaction-scheduler-bench/src/main.rs b/transaction-scheduler-bench/src/main.rs
index c7fd0d36fd2503..5aaf0c860ee7d2 100644
--- a/transaction-scheduler-bench/src/main.rs
+++ b/transaction-scheduler-bench/src/main.rs
@@ -134,7 +134,7 @@ fn spawn_unified_scheduler(
             &mut runnable_queue,
             &mut contended_queue,
             &mut address_book,
-            packet_batch_receiver.clone(),
+            &packet_batch_receiver.clone(),
             &transaction_batch_senders[0],
             //&completed_transaction_receiver
         );

From b65c396bd3c64312989ea78db794e40ff7049ba3 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 5 Aug 2022 23:24:39 +0900
Subject: [PATCH 0090/3199] save

---
 transaction-scheduler-bench/src/main.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/transaction-scheduler-bench/src/main.rs b/transaction-scheduler-bench/src/main.rs
index 5aaf0c860ee7d2..3dd400354ae5a8 100644
--- a/transaction-scheduler-bench/src/main.rs
+++ b/transaction-scheduler-bench/src/main.rs
@@ -124,7 +124,7 @@ fn spawn_unified_scheduler(
         max_batch_size: usize,
         exit: Arc<AtomicBool>,
 ) -> JoinHandle<()> {
-    std::thread::spawn(|| {
+    std::thread::spawn(move || {
         let mut runnable_queue = solana_scheduler::TaskQueue::default();
         let mut contended_queue = solana_scheduler::TaskQueue::default();
         let mut address_book = solana_scheduler::AddressBook::default();

From a1143963f6e03026533b15524d75d966690aec6b Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 5 Aug 2022 23:29:11 +0900
Subject: [PATCH 0091/3199] save

---
 transaction-scheduler-bench/src/main.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/transaction-scheduler-bench/src/main.rs b/transaction-scheduler-bench/src/main.rs
index 3dd400354ae5a8..48fbf3a02df4d3 100644
--- a/transaction-scheduler-bench/src/main.rs
+++ b/transaction-scheduler-bench/src/main.rs
@@ -124,7 +124,7 @@ fn spawn_unified_scheduler(
         max_batch_size: usize,
         exit: Arc<AtomicBool>,
 ) -> JoinHandle<()> {
-    std::thread::spawn(move || {
+    std::thread::Builder::new().name("sol-scheduler".to_string()).spawn(move || {
         let mut runnable_queue = solana_scheduler::TaskQueue::default();
         let mut contended_queue = solana_scheduler::TaskQueue::default();
         let mut address_book = solana_scheduler::AddressBook::default();
@@ -138,7 +138,7 @@ fn spawn_unified_scheduler(
             &transaction_batch_senders[0],
             //&completed_transaction_receiver
         );
-    })
+    }).unwrap()
 }
 
 fn main() {

From cac9ed1ec76ec6cbbd5be5c8d68ba1604c87e3d3 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 5 Aug 2022 23:30:54 +0900
Subject: [PATCH 0092/3199] save

---
 transaction-scheduler-bench/src/main.rs | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/transaction-scheduler-bench/src/main.rs b/transaction-scheduler-bench/src/main.rs
index 48fbf3a02df4d3..890e1204919a05 100644
--- a/transaction-scheduler-bench/src/main.rs
+++ b/transaction-scheduler-bench/src/main.rs
@@ -210,7 +210,7 @@ fn main() {
     );
 
     // Spawn thread for reporting metrics
-    std::thread::spawn({
+    std::thread::Builder::new().name("sol-metrics").spawn({
         move || {
             let start = Instant::now();
             loop {
@@ -231,7 +231,7 @@ fn main() {
             }
             exit.store(true, Ordering::Relaxed);
         }
-    });
+    }).unwrap();
 
     scheduler_handle.join().unwrap();
     execution_handles
@@ -273,7 +273,7 @@ fn start_execution_thread(
     execution_per_tx_us: u64,
     exit: Arc<AtomicBool>,
 ) -> JoinHandle<()> {
-    std::thread::spawn(move || {
+    std::thread::Builder::new().name(format!("blockstore_processor_{}", thread_index).spawn(move || {
         execution_worker(
             metrics,
             thread_index,
@@ -282,7 +282,7 @@ fn start_execution_thread(
             execution_per_tx_us,
             exit,
         )
-    })
+    }).unwrap()
 }
 
 fn execution_worker(

From 5003df15aa443cc54d8b274d7fc2122a384457df Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 5 Aug 2022 23:31:19 +0900
Subject: [PATCH 0093/3199] save

---
 transaction-scheduler-bench/src/main.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/transaction-scheduler-bench/src/main.rs b/transaction-scheduler-bench/src/main.rs
index 890e1204919a05..66b5862baf640b 100644
--- a/transaction-scheduler-bench/src/main.rs
+++ b/transaction-scheduler-bench/src/main.rs
@@ -273,7 +273,7 @@ fn start_execution_thread(
     execution_per_tx_us: u64,
     exit: Arc<AtomicBool>,
 ) -> JoinHandle<()> {
-    std::thread::Builder::new().name(format!("blockstore_processor_{}", thread_index).spawn(move || {
+    std::thread::Builder::new().name(format!("blockstore_processor_{}", thread_index)).spawn(move || {
         execution_worker(
             metrics,
             thread_index,

From 520b6f5125d4863f3449288b6b9b15e1adcf0997 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 5 Aug 2022 23:31:32 +0900
Subject: [PATCH 0094/3199] save

---
 transaction-scheduler-bench/src/main.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/transaction-scheduler-bench/src/main.rs b/transaction-scheduler-bench/src/main.rs
index 66b5862baf640b..e5cca3dca398fe 100644
--- a/transaction-scheduler-bench/src/main.rs
+++ b/transaction-scheduler-bench/src/main.rs
@@ -210,7 +210,7 @@ fn main() {
     );
 
     // Spawn thread for reporting metrics
-    std::thread::Builder::new().name("sol-metrics").spawn({
+    std::thread::Builder::new().name("sol-metrics".to_string()).spawn({
         move || {
             let start = Instant::now();
             loop {

From 1b5542b19e3903644278d41f7976c2dc6821ea60 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 5 Aug 2022 23:32:21 +0900
Subject: [PATCH 0095/3199] save

---
 transaction-scheduler-bench/src/main.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/transaction-scheduler-bench/src/main.rs b/transaction-scheduler-bench/src/main.rs
index e5cca3dca398fe..4e7358f778ef1a 100644
--- a/transaction-scheduler-bench/src/main.rs
+++ b/transaction-scheduler-bench/src/main.rs
@@ -379,7 +379,7 @@ fn spawn_packet_sender(
     duration: Duration,
     exit: Arc<AtomicBool>,
 ) -> JoinHandle<()> {
-    std::thread::spawn(move || {
+    std::thread::Builder::new().name("sol-producer").spawn(move || {
         send_packets(
             metrics,
             num_accounts,
@@ -389,7 +389,7 @@ fn spawn_packet_sender(
             duration,
             exit,
         );
-    })
+    }).unwrap()
 }
 
 fn send_packets(

From 2530b902cc7be4bc773968004c47c36e90381b50 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 5 Aug 2022 23:32:48 +0900
Subject: [PATCH 0096/3199] save

---
 transaction-scheduler-bench/src/main.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/transaction-scheduler-bench/src/main.rs b/transaction-scheduler-bench/src/main.rs
index 4e7358f778ef1a..aa7a24f7124afb 100644
--- a/transaction-scheduler-bench/src/main.rs
+++ b/transaction-scheduler-bench/src/main.rs
@@ -379,7 +379,7 @@ fn spawn_packet_sender(
     duration: Duration,
     exit: Arc<AtomicBool>,
 ) -> JoinHandle<()> {
-    std::thread::Builder::new().name("sol-producer").spawn(move || {
+    std::thread::Builder::new().name("sol-producer".to_string()).spawn(move || {
         send_packets(
             metrics,
             num_accounts,

From 8ddebef074256480c64ed7267a71aeff4a3bb01f Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 5 Aug 2022 23:39:35 +0900
Subject: [PATCH 0097/3199] save

---
 transaction-scheduler-bench/src/main.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/transaction-scheduler-bench/src/main.rs b/transaction-scheduler-bench/src/main.rs
index aa7a24f7124afb..39176e7f8da9c2 100644
--- a/transaction-scheduler-bench/src/main.rs
+++ b/transaction-scheduler-bench/src/main.rs
@@ -478,7 +478,7 @@ fn build_packet(
             read_account_metas.chain(write_account_metas).collect(),
         ),
     ];
-    let versioned_transaction = VersionedTransaction::from(Transaction::new_signed_with_payer(
+    let versioned_transaction = VersionedTransaction::from(Transaction::new_with_payer(
         &ixs,
         Some(&sending_keypair.pubkey()),
         &[sending_keypair],

From 559b39c131c6f913ef6c794aec0b12a312fb96d6 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 5 Aug 2022 23:40:03 +0900
Subject: [PATCH 0098/3199] save

---
 transaction-scheduler-bench/src/main.rs | 2 --
 1 file changed, 2 deletions(-)

diff --git a/transaction-scheduler-bench/src/main.rs b/transaction-scheduler-bench/src/main.rs
index 39176e7f8da9c2..0ff5a8f3341851 100644
--- a/transaction-scheduler-bench/src/main.rs
+++ b/transaction-scheduler-bench/src/main.rs
@@ -481,8 +481,6 @@ fn build_packet(
     let versioned_transaction = VersionedTransaction::from(Transaction::new_with_payer(
         &ixs,
         Some(&sending_keypair.pubkey()),
-        &[sending_keypair],
-        *blockhash,
     ));
 
     let sanitized_tx = SanitizedTransaction::try_create(

From 67cebc65ded72f393550f27e14916168fbd5ebf9 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 5 Aug 2022 23:44:44 +0900
Subject: [PATCH 0099/3199] save

---
 transaction-scheduler-bench/src/main.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/transaction-scheduler-bench/src/main.rs b/transaction-scheduler-bench/src/main.rs
index 0ff5a8f3341851..4ba288eaf1546e 100644
--- a/transaction-scheduler-bench/src/main.rs
+++ b/transaction-scheduler-bench/src/main.rs
@@ -273,7 +273,7 @@ fn start_execution_thread(
     execution_per_tx_us: u64,
     exit: Arc<AtomicBool>,
 ) -> JoinHandle<()> {
-    std::thread::Builder::new().name(format!("blockstore_processor_{}", thread_index)).spawn(move || {
+    std::thread::Builder::new().name(format!("sol-exec-{}", thread_index)).spawn(move || {
         execution_worker(
             metrics,
             thread_index,

From 1af1d870152fe59fd6bb6b115dadc2f79ae29cd6 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 5 Aug 2022 23:45:33 +0900
Subject: [PATCH 0100/3199] save

---
 transaction-scheduler-bench/src/main.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/transaction-scheduler-bench/src/main.rs b/transaction-scheduler-bench/src/main.rs
index 4ba288eaf1546e..6293756913ca39 100644
--- a/transaction-scheduler-bench/src/main.rs
+++ b/transaction-scheduler-bench/src/main.rs
@@ -339,7 +339,7 @@ fn handle_transaction_batch(
             .unwrap();
 }
 
-const NUM_SENDERS: usize = 4;
+const NUM_SENDERS: usize = 2;
 
 fn spawn_packet_senders(
     metrics: Arc<TransactionSchedulerBenchMetrics>,

From 790240e826558bea9007af4cfb370c605d6326aa Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 5 Aug 2022 23:49:00 +0900
Subject: [PATCH 0101/3199] save

---
 transaction-scheduler-bench/src/main.rs | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/transaction-scheduler-bench/src/main.rs b/transaction-scheduler-bench/src/main.rs
index 6293756913ca39..572d2ab14fd8a7 100644
--- a/transaction-scheduler-bench/src/main.rs
+++ b/transaction-scheduler-bench/src/main.rs
@@ -513,10 +513,10 @@ fn build_accounts(num_accounts: usize) -> Vec<Keypair> {
 fn build_channels<T>(num_execution_threads: usize) -> (Vec<Sender<T>>, Vec<Receiver<T>>) {
     let mut senders = Vec::with_capacity(num_execution_threads);
     let mut receivers = Vec::with_capacity(num_execution_threads);
+    let (sender, receiver) = crossbeam_channel::unbounded();
     for _ in 0..num_execution_threads {
-        let (sender, receiver) = crossbeam_channel::unbounded();
-        senders.push(sender);
-        receivers.push(receiver);
+        senders.push(sender.clone());
+        receivers.push(receiver.clone());
     }
     (senders, receivers)
 }

From e235a17dc6be1a3b91ec43315d5efd95c8283370 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 5 Aug 2022 23:51:07 +0900
Subject: [PATCH 0102/3199] save

---
 transaction-scheduler-bench/src/main.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/transaction-scheduler-bench/src/main.rs b/transaction-scheduler-bench/src/main.rs
index 572d2ab14fd8a7..15ddb048f36675 100644
--- a/transaction-scheduler-bench/src/main.rs
+++ b/transaction-scheduler-bench/src/main.rs
@@ -339,7 +339,7 @@ fn handle_transaction_batch(
             .unwrap();
 }
 
-const NUM_SENDERS: usize = 2;
+const NUM_SENDERS: usize = 1;
 
 fn spawn_packet_senders(
     metrics: Arc<TransactionSchedulerBenchMetrics>,

From c8cb4c47e901722f1bf141c0c652ae4c02939d5e Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 5 Aug 2022 23:56:46 +0900
Subject: [PATCH 0103/3199] save

---
 scheduler/src/lib.rs | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index a7abec2a022b90..a6ac3a7965a0d0 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -624,6 +624,16 @@ impl ScheduleStage {
                         for v in vv {
                             let p = get_transaction_priority_details(&v.0);
                             Self::register_runnable_task((Weight { ix: p }, v), runnable_queue);
+                            while executing_queue_count < max_executing_queue_count {
+                                let maybe_ee =
+                                    Self::schedule_next_execution(runnable_queue, contended_queue, address_book);
+                                if let Some(ee) = maybe_ee {
+                                    trace!("send to execute");
+                                    executing_queue_count += 1;
+
+                                    to_execute_substage.send(ee).unwrap();
+                                }
+                            }
                         }
                     }
                 }

From f4fd2a4f9770339c459f4cad56f2180080994751 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 5 Aug 2022 23:57:50 +0900
Subject: [PATCH 0104/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index a6ac3a7965a0d0..f31821437eb2be 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -624,7 +624,7 @@ impl ScheduleStage {
                         for v in vv {
                             let p = get_transaction_priority_details(&v.0);
                             Self::register_runnable_task((Weight { ix: p }, v), runnable_queue);
-                            while executing_queue_count < max_executing_queue_count {
+                            if executing_queue_count < max_executing_queue_count {
                                 let maybe_ee =
                                     Self::schedule_next_execution(runnable_queue, contended_queue, address_book);
                                 if let Some(ee) = maybe_ee {

From 15606d0fa44b194954eb33fee843e0c947a008fe Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 6 Aug 2022 00:03:14 +0900
Subject: [PATCH 0105/3199] save

---
 scheduler/src/lib.rs | 16 ++++++++++------
 1 file changed, 10 insertions(+), 6 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index f31821437eb2be..49f14c45ba1c60 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -350,9 +350,10 @@ impl ScheduleStage {
     fn push_to_queue(
         (weight, tx): (Weight, Box<(SanitizedTransaction, Vec<LockAttempt>)>),
         runnable_queue: &mut TaskQueue,
+        &mut unique_key,
     ) {
         // manage randomness properly for future scheduling determinism
-        let mut rng = rand::thread_rng();
+        //let mut rng = rand::thread_rng();
 
         //let ix = 23;
         //let tx = bank
@@ -366,10 +367,12 @@ impl ScheduleStage {
         runnable_queue.add_to_schedule(
             UniqueWeight {
                 weight,
-                unique_key: solana_sdk::hash::new_rand(&mut rng),
+                //unique_key: solana_sdk::hash::new_rand(&mut rng),
+                unique_key,
             },
             Task { tx },
         );
+        unique_key -= 1;
     }
 
     #[inline(never)]
@@ -591,8 +594,9 @@ impl ScheduleStage {
     fn register_runnable_task(
         weighted_tx: (Weight, Box<(SanitizedTransaction, Vec<LockAttempt>)>),
         runnable_queue: &mut TaskQueue,
+        unique_key: &mut u64,
     ) {
-        Self::push_to_queue(weighted_tx, runnable_queue)
+        Self::push_to_queue(weighted_tx, runnable_queue, unique_key)
     }
 
     pub fn run(
@@ -605,6 +609,7 @@ impl ScheduleStage {
         // to_next_stage: &crossbeam_channel::Sender<Box<ExecutionEnvironment>>, // assume nonblocking
     ) {
         let mut executing_queue_count = 0;
+        let mut current_unique_key = u64::max_value();
 
         loop {
             trace!("schedule_once!");
@@ -614,16 +619,15 @@ impl ScheduleStage {
                 MultiplexedPayload::FromPrevious(weighted_tx) => {
                     trace!("recv from previous");
 
-                    Self::register_runnable_task(weighted_tx, runnable_queue);
+                    Self::register_runnable_task(weighted_tx, runnable_queue, &mut current_unique_key);
                 }
                 MultiplexedPayload::FromPreviousBatched(vvv) => {
                     trace!("recv from previous");
 
-                    //Self::register_runnable_task(weighted_tx, runnable_queue);
                     for vv in vvv {
                         for v in vv {
                             let p = get_transaction_priority_details(&v.0);
-                            Self::register_runnable_task((Weight { ix: p }, v), runnable_queue);
+                            Self::register_runnable_task((Weight { ix: p }, v), runnable_queue, &mut current_unique_key);
                             if executing_queue_count < max_executing_queue_count {
                                 let maybe_ee =
                                     Self::schedule_next_execution(runnable_queue, contended_queue, address_book);

From 8d697956578f9ad519dd726e15e14a3d54e7a4ac Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 6 Aug 2022 00:03:28 +0900
Subject: [PATCH 0106/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 49f14c45ba1c60..c1d8efecaaf0b4 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -350,7 +350,7 @@ impl ScheduleStage {
     fn push_to_queue(
         (weight, tx): (Weight, Box<(SanitizedTransaction, Vec<LockAttempt>)>),
         runnable_queue: &mut TaskQueue,
-        &mut unique_key,
+        &mut unique_key: u64,
     ) {
         // manage randomness properly for future scheduling determinism
         //let mut rng = rand::thread_rng();

From 46d276e6621cae38e8cd248b4994acee44b30295 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 6 Aug 2022 00:03:47 +0900
Subject: [PATCH 0107/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index c1d8efecaaf0b4..057a5a56506e37 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -350,7 +350,7 @@ impl ScheduleStage {
     fn push_to_queue(
         (weight, tx): (Weight, Box<(SanitizedTransaction, Vec<LockAttempt>)>),
         runnable_queue: &mut TaskQueue,
-        &mut unique_key: u64,
+        unique_key: &mut u64,
     ) {
         // manage randomness properly for future scheduling determinism
         //let mut rng = rand::thread_rng();

From b0ff6b413e37e2fa8b3535d24badd1fe9d8fb7d4 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 6 Aug 2022 00:04:02 +0900
Subject: [PATCH 0108/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 057a5a56506e37..58f21e19489700 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -372,7 +372,7 @@ impl ScheduleStage {
             },
             Task { tx },
         );
-        unique_key -= 1;
+        *unique_key -= 1;
     }
 
     #[inline(never)]

From 3ac82ff13ecfc8e6806521ca46441c4c70254e1c Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 6 Aug 2022 00:04:18 +0900
Subject: [PATCH 0109/3199] save

---
 scheduler/src/lib.rs | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 58f21e19489700..88e6581a2baf60 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -248,7 +248,8 @@ pub struct UniqueWeight {
     weight: Weight,
     // we can't use Transaction::message_hash because it's manipulatable to be favorous to the tx
     // submitter
-    unique_key: Hash, // tie breaker? random noise? also for unique identification of txes?
+    //unique_key: Hash, // tie breaker? random noise? also for unique identification of txes?
+    unique_key: u64, // tie breaker? random noise? also for unique identification of txes?
     // fee?
 }
 /*

From 5436ce09a2c362a48afa1eb897ee6c812035bcfe Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 6 Aug 2022 00:04:36 +0900
Subject: [PATCH 0110/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 88e6581a2baf60..d4a1fb0d2e5333 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -369,7 +369,7 @@ impl ScheduleStage {
             UniqueWeight {
                 weight,
                 //unique_key: solana_sdk::hash::new_rand(&mut rng),
-                unique_key,
+                unique_key: *unique_key,
             },
             Task { tx },
         );

From af77133d7b83fe042d9fdc4bf89d6484870c6048 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 6 Aug 2022 22:10:14 +0900
Subject: [PATCH 0111/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index d4a1fb0d2e5333..2319b6f3aebe13 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -107,7 +107,7 @@ impl Page {
     }
 }
 
-type AddressMap = std::collections::HashMap<Pubkey, Page>;
+type AddressMap = std::collections::HashMap<Pubkey, Rc<Page>>;
 type AddressSet = std::collections::HashSet<Pubkey>;
 type AddressMapEntry<'a, K, V> = std::collections::hash_map::Entry<'a, K, V>;
 

From ff571c3ee3a2576396d329251ab1d5099f65c2ab Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 6 Aug 2022 22:10:46 +0900
Subject: [PATCH 0112/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 2319b6f3aebe13..70c219280d76ea 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -107,7 +107,7 @@ impl Page {
     }
 }
 
-type AddressMap = std::collections::HashMap<Pubkey, Rc<Page>>;
+type AddressMap = std::collections::HashMap<Pubkey, std::rc::Rc<Page>>;
 type AddressSet = std::collections::HashSet<Pubkey>;
 type AddressMapEntry<'a, K, V> = std::collections::hash_map::Entry<'a, K, V>;
 

From 2f0bc932975b838e7f7577aec577d4ff5642f7a3 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 6 Aug 2022 22:11:14 +0900
Subject: [PATCH 0113/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 70c219280d76ea..130a3653c3be06 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -130,7 +130,7 @@ impl AddressBook {
 
         match self.book.entry(*address) {
             AddressMapEntry::Vacant(book_entry) => {
-                book_entry.insert(Page::new(CurrentUsage::renew(*requested_usage)));
+                book_entry.insert(Rc::new(Page::new(CurrentUsage::renew(*requested_usage))));
                 *is_success = true;
             }
             AddressMapEntry::Occupied(mut book_entry) => {

From ea851ae1186c6c25fea64f90cf41f80ff7a5f939 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 6 Aug 2022 22:11:42 +0900
Subject: [PATCH 0114/3199] save

---
 scheduler/src/lib.rs | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 130a3653c3be06..53f64d04a69541 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -16,6 +16,8 @@ use {
     },
 };
 
+use std::rc::Rc;
+
 #[derive(Debug)]
 pub struct ExecutionEnvironment {
     lock_attempts: Vec<LockAttempt>,

From bd4e4eed7570e7238ad96c00130dd5cb72103e07 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 6 Aug 2022 22:12:25 +0900
Subject: [PATCH 0115/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 53f64d04a69541..45c3d1f340322f 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -136,7 +136,7 @@ impl AddressBook {
                 *is_success = true;
             }
             AddressMapEntry::Occupied(mut book_entry) => {
-                let mut page = book_entry.get_mut();
+                let mut page = *book_entry.get_mut();
 
                 match &mut page.current_usage {
                     CurrentUsage::Unused => {

From c0e27b0029cfac152ca6badf07624cf72fd0bf4b Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 6 Aug 2022 22:12:59 +0900
Subject: [PATCH 0116/3199] save

---
 scheduler/src/lib.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 45c3d1f340322f..49d748e42fb150 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -150,7 +150,7 @@ impl AddressBook {
                         }
                         RequestedUsage::Writable => {
                             if from_runnable {
-                                Self::remember_address_contention(page, unique_weight);
+                                Self::remember_address_contention(&mut page, unique_weight);
                             }
                             *is_success = false;
                         }
@@ -158,7 +158,7 @@ impl AddressBook {
                     CurrentUsage::Writable => match requested_usage {
                         RequestedUsage::Readonly | RequestedUsage::Writable => {
                             if from_runnable {
-                                Self::remember_address_contention(page, unique_weight);
+                                Self::remember_address_contention(&mut page, unique_weight);
                             }
                             *is_success = false;
                         }

From ac95a35a35125bf8786b687aca0a55bd81961f00 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 6 Aug 2022 22:13:36 +0900
Subject: [PATCH 0117/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 49d748e42fb150..8bf4a61467c250 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -136,7 +136,7 @@ impl AddressBook {
                 *is_success = true;
             }
             AddressMapEntry::Occupied(mut book_entry) => {
-                let mut page = *book_entry.get_mut();
+                let mut page = &*book_entry.get_mut();
 
                 match &mut page.current_usage {
                     CurrentUsage::Unused => {

From 83ba73fdc172c3646c78e1e778165cc9684f6adb Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 6 Aug 2022 22:14:22 +0900
Subject: [PATCH 0118/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 8bf4a61467c250..917efe04fb4f7f 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -136,7 +136,7 @@ impl AddressBook {
                 *is_success = true;
             }
             AddressMapEntry::Occupied(mut book_entry) => {
-                let mut page = &*book_entry.get_mut();
+                let mut page = book_entry.get_mut().get_mut();
 
                 match &mut page.current_usage {
                     CurrentUsage::Unused => {

From a883648adc489fa038dc21fff6663230f406c935 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 6 Aug 2022 22:15:14 +0900
Subject: [PATCH 0119/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 917efe04fb4f7f..7966766890541c 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -136,7 +136,7 @@ impl AddressBook {
                 *is_success = true;
             }
             AddressMapEntry::Occupied(mut book_entry) => {
-                let mut page = book_entry.get_mut().get_mut();
+                let mut page = Rc::<Page>::get_mut(book_entry.get_mut());
 
                 match &mut page.current_usage {
                     CurrentUsage::Unused => {

From 7b84853f4d325329f965ea120ecf8ae410476e45 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 6 Aug 2022 22:15:36 +0900
Subject: [PATCH 0120/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 7966766890541c..40142af6ae80c7 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -138,7 +138,7 @@ impl AddressBook {
             AddressMapEntry::Occupied(mut book_entry) => {
                 let mut page = Rc::<Page>::get_mut(book_entry.get_mut());
 
-                match &mut page.current_usage {
+                match page.current_usage {
                     CurrentUsage::Unused => {
                         page.current_usage = CurrentUsage::renew(*requested_usage);
                         *is_success = true;

From fbcbf52785f2fdae1d3bd88bce9c97c5db55add5 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 6 Aug 2022 22:16:01 +0900
Subject: [PATCH 0121/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 40142af6ae80c7..29ff8730a9a548 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -136,7 +136,7 @@ impl AddressBook {
                 *is_success = true;
             }
             AddressMapEntry::Occupied(mut book_entry) => {
-                let mut page = Rc::<Page>::get_mut(book_entry.get_mut());
+                let mut page = Rc::<Page>::get_mut(book_entry.get_mut()).unwrap();
 
                 match page.current_usage {
                     CurrentUsage::Unused => {

From d9e9e2735ec4db99e0686704235be9a1ad3fd2df Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 6 Aug 2022 22:16:41 +0900
Subject: [PATCH 0122/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 29ff8730a9a548..d5b98ec4556ca1 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -179,7 +179,7 @@ impl AddressBook {
             AddressMapEntry::Vacant(_book_entry) => unreachable!(),
             AddressMapEntry::Occupied(mut entry) => {
                 let page = entry.get_mut();
-                page.contended_unique_weights.remove(unique_weight);
+                *page.contended_unique_weights.remove(unique_weight);
             }
         }
     }

From dff2ccf9cc0103c0d032eac239ba5f4202fbcacb Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 6 Aug 2022 22:17:05 +0900
Subject: [PATCH 0123/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index d5b98ec4556ca1..874f4034caece3 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -179,7 +179,7 @@ impl AddressBook {
             AddressMapEntry::Vacant(_book_entry) => unreachable!(),
             AddressMapEntry::Occupied(mut entry) => {
                 let page = entry.get_mut();
-                *page.contended_unique_weights.remove(unique_weight);
+                Rc::get_mut(page).contended_unique_weights.remove(unique_weight);
             }
         }
     }

From 901b41b494696f10f1109673df77061d96182522 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 6 Aug 2022 22:17:24 +0900
Subject: [PATCH 0124/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 874f4034caece3..54108b68158b5b 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -179,7 +179,7 @@ impl AddressBook {
             AddressMapEntry::Vacant(_book_entry) => unreachable!(),
             AddressMapEntry::Occupied(mut entry) => {
                 let page = entry.get_mut();
-                Rc::get_mut(page).contended_unique_weights.remove(unique_weight);
+                Rc::get_mut(page).unwrap().contended_unique_weights.remove(unique_weight);
             }
         }
     }

From eddea516248083c274b6a212e6153645e44c8d11 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 6 Aug 2022 22:17:58 +0900
Subject: [PATCH 0125/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 54108b68158b5b..beb14df21a015e 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -199,7 +199,7 @@ impl AddressBook {
 
         match self.book.entry(attempt.address) {
             AddressMapEntry::Occupied(mut book_entry) => {
-                let mut page = book_entry.get_mut();
+                let mut page = Rc::get_mut(book_entry.get_mut()).unwrap();
 
                 match &mut page.current_usage {
                     CurrentUsage::Readonly(ref mut count) => match &attempt.requested_usage {

From 5af074d41e733380e37c62c628dac57852fbca0d Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 6 Aug 2022 22:29:51 +0900
Subject: [PATCH 0126/3199] save

---
 scheduler/src/lib.rs | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index beb14df21a015e..f87fd0026bd4b6 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -39,6 +39,11 @@ impl ExecutionEnvironment {
     //}
 }
 
+enum LockAttemptStatus {
+    BeforeLookup(Pubkey),
+    AfterLookup(Rc<Page>),
+}
+
 #[derive(Clone, Debug)]
 pub struct LockAttempt {
     address: Pubkey,

From 98caaaefa8e786c2175ce4f3e7b685b5b6ac3690 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 6 Aug 2022 22:30:20 +0900
Subject: [PATCH 0127/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index f87fd0026bd4b6..b8213e90bd1a25 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -46,7 +46,7 @@ enum LockAttemptStatus {
 
 #[derive(Clone, Debug)]
 pub struct LockAttempt {
-    address: Pubkey,
+    status: LockAttemptStatus,
     is_success: bool,
     requested_usage: RequestedUsage,
 }

From 582068f29d353e5748581a36e595369c26a14959 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 6 Aug 2022 22:30:45 +0900
Subject: [PATCH 0128/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index b8213e90bd1a25..38173ccf16d1e1 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -62,7 +62,7 @@ impl LockAttempt {
 
     pub fn new(address: Pubkey, requested_usage: RequestedUsage) -> Self {
         Self {
-            address,
+            status: LockAttemptStatus::BeforeLookup(address),
             is_success: true,
             requested_usage,
         }

From 06edca55b58318ddb61af64e85d0ff15557deedc Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 6 Aug 2022 22:31:38 +0900
Subject: [PATCH 0129/3199] save

---
 scheduler/src/lib.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 38173ccf16d1e1..2f1ffc92a5267b 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -133,9 +133,9 @@ impl AddressBook {
         unique_weight: &UniqueWeight,
         attempt: &mut LockAttempt,
     ) {
-        let LockAttempt {address, requested_usage, is_success} = attempt;
+        let LockAttempt {status, requested_usage, is_success} = attempt;
 
-        match self.book.entry(*address) {
+        match self.book.entry(status.address()) {
             AddressMapEntry::Vacant(book_entry) => {
                 book_entry.insert(Rc::new(Page::new(CurrentUsage::renew(*requested_usage))));
                 *is_success = true;

From e0bcd488cf0f2c14b3123ca5c1b127359261b8ff Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 6 Aug 2022 22:33:25 +0900
Subject: [PATCH 0130/3199] save

---
 scheduler/src/lib.rs | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 2f1ffc92a5267b..df32274baa0ff0 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -39,11 +39,21 @@ impl ExecutionEnvironment {
     //}
 }
 
+#[derive(Clone, Debug)]
 enum LockAttemptStatus {
     BeforeLookup(Pubkey),
     AfterLookup(Rc<Page>),
 }
 
+impl LockAttemptStatus {
+    fn address(&self) -> &Pubkey {
+        match self {
+            LockAttemptStatus::BeforeLookup(pubkey) => &pubkey,
+            LockAttemptStatus::AfterLookup(_) => unreachable!(),
+        }
+    }
+}
+
 #[derive(Clone, Debug)]
 pub struct LockAttempt {
     status: LockAttemptStatus,

From 0516d7d9981b8c2a12c4e39ec79bc498b880f825 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 6 Aug 2022 22:33:40 +0900
Subject: [PATCH 0131/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index df32274baa0ff0..1a9e3e1ac5989c 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -145,7 +145,7 @@ impl AddressBook {
     ) {
         let LockAttempt {status, requested_usage, is_success} = attempt;
 
-        match self.book.entry(status.address()) {
+        match self.book.entry(*status.address()) {
             AddressMapEntry::Vacant(book_entry) => {
                 book_entry.insert(Rc::new(Page::new(CurrentUsage::renew(*requested_usage))));
                 *is_success = true;

From 986a84959fb95e3fad2c05cc761f9f1a9c592771 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 6 Aug 2022 22:34:15 +0900
Subject: [PATCH 0132/3199] save

---
 scheduler/src/lib.rs | 1 +
 1 file changed, 1 insertion(+)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 1a9e3e1ac5989c..a232721903b1b6 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -106,6 +106,7 @@ pub enum RequestedUsage {
     Writable,
 }
 
+#[derive(Debug)]
 struct Page {
     current_usage: CurrentUsage,
     contended_unique_weights: std::collections::BTreeSet<UniqueWeight>,

From da09c91f2fac6cac77dd5e2306a3feb26a94f8aa Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 6 Aug 2022 22:35:58 +0900
Subject: [PATCH 0133/3199] save

---
 scheduler/src/lib.rs | 55 +++++++++++++++++++-------------------------
 1 file changed, 24 insertions(+), 31 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index a232721903b1b6..1640d8262f1d6a 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -213,39 +213,32 @@ impl AddressBook {
         let mut newly_uncontended = false;
         let mut still_queued = false;
 
-        match self.book.entry(attempt.address) {
-            AddressMapEntry::Occupied(mut book_entry) => {
-                let mut page = Rc::get_mut(book_entry.get_mut()).unwrap();
-
-                match &mut page.current_usage {
-                    CurrentUsage::Readonly(ref mut count) => match &attempt.requested_usage {
-                        RequestedUsage::Readonly => {
-                            if *count == SOLE_USE_COUNT {
-                                newly_uncontended = true;
-                            } else {
-                                *count -= 1;
-                            }
-                        }
-                        RequestedUsage::Writable => unreachable!(),
-                    },
-                    CurrentUsage::Writable => match &attempt.requested_usage {
-                        RequestedUsage::Writable => {
-                            newly_uncontended = true;
-                        }
-                        RequestedUsage::Readonly => unreachable!(),
-                    },
-                    CurrentUsage::Unused => unreachable!(),
-                }
-
-                if newly_uncontended {
-                    page.current_usage = CurrentUsage::Unused;
-                    if !page.contended_unique_weights.is_empty() {
-                        still_queued = true;
+        let mut page = attempt.page();
+
+        match &mut page.current_usage {
+            CurrentUsage::Readonly(ref mut count) => match &attempt.requested_usage {
+                RequestedUsage::Readonly => {
+                    if *count == SOLE_USE_COUNT {
+                        newly_uncontended = true;
+                    } else {
+                        *count -= 1;
                     }
                 }
-            }
-            AddressMapEntry::Vacant(_book_entry) => {
-                unreachable!()
+                RequestedUsage::Writable => unreachable!(),
+            },
+            CurrentUsage::Writable => match &attempt.requested_usage {
+                RequestedUsage::Writable => {
+                    newly_uncontended = true;
+                }
+                RequestedUsage::Readonly => unreachable!(),
+            },
+            CurrentUsage::Unused => unreachable!(),
+        }
+
+        if newly_uncontended {
+            page.current_usage = CurrentUsage::Unused;
+            if !page.contended_unique_weights.is_empty() {
+                still_queued = true;
             }
         }
 

From 34c0f8ad902b2e9ded344381be050d389125cbbb Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 6 Aug 2022 22:36:11 +0900
Subject: [PATCH 0134/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 1640d8262f1d6a..695f7b4fcbe025 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -213,7 +213,7 @@ impl AddressBook {
         let mut newly_uncontended = false;
         let mut still_queued = false;
 
-        let mut page = attempt.page();
+        let mut page = attempt.status.page();
 
         match &mut page.current_usage {
             CurrentUsage::Readonly(ref mut count) => match &attempt.requested_usage {

From 590143b568b333ee2dd4aba40808e631599ba43b Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 6 Aug 2022 22:37:12 +0900
Subject: [PATCH 0135/3199] save

---
 scheduler/src/lib.rs | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 695f7b4fcbe025..12100595495572 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -52,6 +52,13 @@ impl LockAttemptStatus {
             LockAttemptStatus::AfterLookup(_) => unreachable!(),
         }
     }
+
+    fn page(&self) -> Rc<Page> {
+        match self {
+            LockAttemptStatus::BeforeLookup(_) => unreachable!(),
+            LockAttemptStatus::AfterLookup(page) => Rc::get_mut(page).unwrap(),
+        }
+    }
 }
 
 #[derive(Clone, Debug)]

From 67bf1797a4c1e45e389b998080c441b703e08f21 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 6 Aug 2022 22:37:49 +0900
Subject: [PATCH 0136/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 12100595495572..5bbc85834456fc 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -53,7 +53,7 @@ impl LockAttemptStatus {
         }
     }
 
-    fn page(&self) -> Rc<Page> {
+    fn page(&mut self) -> Rc<Page> {
         match self {
             LockAttemptStatus::BeforeLookup(_) => unreachable!(),
             LockAttemptStatus::AfterLookup(page) => Rc::get_mut(page).unwrap(),

From 58dd87e7c519f0d3558605acaa122e16dc3ad2fc Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 6 Aug 2022 22:38:10 +0900
Subject: [PATCH 0137/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 5bbc85834456fc..cf18cf8ee9eaf9 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -53,7 +53,7 @@ impl LockAttemptStatus {
         }
     }
 
-    fn page(&mut self) -> Rc<Page> {
+    fn page(&mut self) -> &mut Page {
         match self {
             LockAttemptStatus::BeforeLookup(_) => unreachable!(),
             LockAttemptStatus::AfterLookup(page) => Rc::get_mut(page).unwrap(),

From eaeed63e2885b3a29a9b045fdf5ea6c5b9cb6028 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 6 Aug 2022 22:40:29 +0900
Subject: [PATCH 0138/3199] save

---
 scheduler/src/lib.rs | 12 +++---------
 1 file changed, 3 insertions(+), 9 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index cf18cf8ee9eaf9..1d38cdd44081e2 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -197,14 +197,8 @@ impl AddressBook {
     }
 
     #[inline(never)]
-    fn forget_address_contention(&mut self, unique_weight: &UniqueWeight, address: &Pubkey) {
-        match self.book.entry(*address) {
-            AddressMapEntry::Vacant(_book_entry) => unreachable!(),
-            AddressMapEntry::Occupied(mut entry) => {
-                let page = entry.get_mut();
-                Rc::get_mut(page).unwrap().contended_unique_weights.remove(unique_weight);
-            }
-        }
+    fn forget_address_contention(&mut self, unique_weight: &UniqueWeight, a: &mut LockAttempt) {
+        a.page().contended_unique_weights.remove(unique_weight);
     }
 
     fn ensure_unlock(&mut self, attempt: &LockAttempt) {
@@ -529,7 +523,7 @@ impl ScheduleStage {
     ) {
         for l in lock_attempts {
             // ensure to remove remaining refs of this unique_weight
-            address_book.forget_address_contention(&unique_weight, &l.address);
+            address_book.forget_address_contention(&unique_weight, &mut l);
 
             // revert because now contended again
             address_book.newly_uncontended_addresses.remove(&l.address);

From 1c73f0636b54e82c7535399120591730b5e25721 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 6 Aug 2022 22:40:43 +0900
Subject: [PATCH 0139/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 1d38cdd44081e2..fd3904d4164747 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -198,7 +198,7 @@ impl AddressBook {
 
     #[inline(never)]
     fn forget_address_contention(&mut self, unique_weight: &UniqueWeight, a: &mut LockAttempt) {
-        a.page().contended_unique_weights.remove(unique_weight);
+        a.status.page().contended_unique_weights.remove(unique_weight);
     }
 
     fn ensure_unlock(&mut self, attempt: &LockAttempt) {

From 68d4a3f454eda8c52ba6832ea55451c3cd9432e7 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 6 Aug 2022 22:41:18 +0900
Subject: [PATCH 0140/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index fd3904d4164747..3d3e7855cc90e7 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -42,7 +42,7 @@ impl ExecutionEnvironment {
 #[derive(Clone, Debug)]
 enum LockAttemptStatus {
     BeforeLookup(Pubkey),
-    AfterLookup(Rc<Page>),
+    AfterLookup(Pubkey, Rc<Page>),
 }
 
 impl LockAttemptStatus {

From 3846892b811f75b22e711d37a41a7da16df1d85c Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 6 Aug 2022 22:41:44 +0900
Subject: [PATCH 0141/3199] save

---
 scheduler/src/lib.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 3d3e7855cc90e7..4ef2ef5592b02f 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -49,14 +49,14 @@ impl LockAttemptStatus {
     fn address(&self) -> &Pubkey {
         match self {
             LockAttemptStatus::BeforeLookup(pubkey) => &pubkey,
-            LockAttemptStatus::AfterLookup(_) => unreachable!(),
+            LockAttemptStatus::AfterLookup(_, _) => unreachable!(),
         }
     }
 
     fn page(&mut self) -> &mut Page {
         match self {
             LockAttemptStatus::BeforeLookup(_) => unreachable!(),
-            LockAttemptStatus::AfterLookup(page) => Rc::get_mut(page).unwrap(),
+            LockAttemptStatus::AfterLookup(_address, page) => Rc::get_mut(page).unwrap(),
         }
     }
 }

From 3e8ef50fdff3461db51dd6ab2605fc1ee80d56ae Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 6 Aug 2022 22:42:12 +0900
Subject: [PATCH 0142/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 4ef2ef5592b02f..e92bd9c5798030 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -49,7 +49,7 @@ impl LockAttemptStatus {
     fn address(&self) -> &Pubkey {
         match self {
             LockAttemptStatus::BeforeLookup(pubkey) => &pubkey,
-            LockAttemptStatus::AfterLookup(_, _) => unreachable!(),
+            LockAttemptStatus::AfterLookup(pubkey, _) => &pubkey,
         }
     }
 

From 700fac6eaec5d75bd85df1ed77f2497f2b218f7b Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 6 Aug 2022 22:43:16 +0900
Subject: [PATCH 0143/3199] save

---
 scheduler/src/lib.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index e92bd9c5798030..78f6b84585ecab 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -526,7 +526,7 @@ impl ScheduleStage {
             address_book.forget_address_contention(&unique_weight, &mut l);
 
             // revert because now contended again
-            address_book.newly_uncontended_addresses.remove(&l.address);
+            address_book.newly_uncontended_addresses.remove(&l.status.address());
         }
     }
 
@@ -541,7 +541,7 @@ impl ScheduleStage {
 
             // revert because now contended again
             if !from_runnable {
-                address_book.newly_uncontended_addresses.remove(&l.address);
+                address_book.newly_uncontended_addresses.remove(&l.status.address());
             }
 
             // todo: mem::forget and panic in LockAttempt::drop()

From eaae4afd1c3122542c1fa4b0a9319fd4c9d8cfb7 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 6 Aug 2022 22:43:32 +0900
Subject: [PATCH 0144/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 78f6b84585ecab..503c7779201128 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -553,7 +553,7 @@ impl ScheduleStage {
         for l in lock_attempts {
             let newly_uncontended_while_queued = address_book.unlock(&l);
             if newly_uncontended_while_queued {
-                address_book.newly_uncontended_addresses.insert(l.address);
+                address_book.newly_uncontended_addresses.insert(&l..status.address());
             }
 
             // todo: mem::forget and panic in LockAttempt::drop()

From 23e6f9fb87b24f584cb8a3111e5f5378e78a3943 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 6 Aug 2022 22:43:45 +0900
Subject: [PATCH 0145/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 503c7779201128..84bd40ca48e3ea 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -553,7 +553,7 @@ impl ScheduleStage {
         for l in lock_attempts {
             let newly_uncontended_while_queued = address_book.unlock(&l);
             if newly_uncontended_while_queued {
-                address_book.newly_uncontended_addresses.insert(&l..status.address());
+                address_book.newly_uncontended_addresses.insert(&l.status.address());
             }
 
             // todo: mem::forget and panic in LockAttempt::drop()

From e99504da7a0638df19ada7b559c473e439634aa0 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 6 Aug 2022 22:44:05 +0900
Subject: [PATCH 0146/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 84bd40ca48e3ea..657177e02b2167 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -89,7 +89,7 @@ impl LockAttempt {
 type UsageCount = usize;
 const SOLE_USE_COUNT: UsageCount = 1;
 
-#[derive(PartialEq)]
+#[derive(Debug, PartialEq)]
 enum CurrentUsage {
     Unused,
     // weight to abort running tx?

From 87f53adb95b119e0115e9b8f9113436d29c9fb8d Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 6 Aug 2022 22:44:21 +0900
Subject: [PATCH 0147/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 657177e02b2167..0ae23d46360a05 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -553,7 +553,7 @@ impl ScheduleStage {
         for l in lock_attempts {
             let newly_uncontended_while_queued = address_book.unlock(&l);
             if newly_uncontended_while_queued {
-                address_book.newly_uncontended_addresses.insert(&l.status.address());
+                address_book.newly_uncontended_addresses.insert(*l.status.address());
             }
 
             // todo: mem::forget and panic in LockAttempt::drop()

From 37ce0bb44db9c991e694d034f1f826c4200542f4 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 6 Aug 2022 22:44:42 +0900
Subject: [PATCH 0148/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 0ae23d46360a05..253f0fef3e9f5e 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -208,7 +208,7 @@ impl AddressBook {
     }
 
     #[inline(never)]
-    fn unlock(&mut self, attempt: &LockAttempt) -> bool {
+    fn unlock(&mut self, attempt: &mut LockAttempt) -> bool {
         debug_assert!(attempt.is_success());
 
         let mut newly_uncontended = false;

From 357ec4388ecb01f32b9611cb6454c13cab468473 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 6 Aug 2022 22:44:54 +0900
Subject: [PATCH 0149/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 253f0fef3e9f5e..2d32601826f668 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -201,7 +201,7 @@ impl AddressBook {
         a.status.page().contended_unique_weights.remove(unique_weight);
     }
 
-    fn ensure_unlock(&mut self, attempt: &LockAttempt) {
+    fn ensure_unlock(&mut self, attempt: &mut LockAttempt) {
         if attempt.is_success() {
             self.unlock(attempt);
         }

From d634748d3fc91bd69400ced294a8680ee68d5b32 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 6 Aug 2022 22:45:19 +0900
Subject: [PATCH 0150/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 2d32601826f668..6ca861e7e52172 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -533,7 +533,7 @@ impl ScheduleStage {
     #[inline(never)]
     fn ensure_unlock_for_failed_execution(
         address_book: &mut AddressBook,
-        lock_attempts: &Vec<LockAttempt>,
+        lock_attempts: &mut Vec<LockAttempt>,
         from_runnable: bool,
     ) {
         for l in lock_attempts {

From 83c8ed0c7ccfdd2d7a1b8afbe5ce15dee66a9a93 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 6 Aug 2022 22:45:38 +0900
Subject: [PATCH 0151/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 6ca861e7e52172..0d1ca7999db6fb 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -537,7 +537,7 @@ impl ScheduleStage {
         from_runnable: bool,
     ) {
         for l in lock_attempts {
-            address_book.ensure_unlock(&l);
+            address_book.ensure_unlock(l);
 
             // revert because now contended again
             if !from_runnable {

From 7971e44cfa6bda492edaba81ee6fc41ca30873b8 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 6 Aug 2022 22:45:54 +0900
Subject: [PATCH 0152/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 0d1ca7999db6fb..2a2ccafc35a0ff 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -489,7 +489,7 @@ impl ScheduleStage {
                 //trace!("ensure_unlock_for_failed_execution(): {:?} {}", (&unique_weight, from_runnable), next_task.tx.signature());
                 Self::ensure_unlock_for_failed_execution(
                     address_book,
-                    &populated_lock_attempts,
+                    &mut populated_lock_attempts,
                     from_runnable,
                 );
                 std::mem::swap(&mut next_task.tx.1, &mut populated_lock_attempts);

From 7dfac2b3e56638d8919cc5f8e181942ed6db06cc Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 6 Aug 2022 22:46:11 +0900
Subject: [PATCH 0153/3199] save

---
 scheduler/src/lib.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 2a2ccafc35a0ff..a0c9dc39239cd7 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -549,9 +549,9 @@ impl ScheduleStage {
     }
 
     #[inline(never)]
-    fn unlock_after_execution(address_book: &mut AddressBook, lock_attempts: Vec<LockAttempt>) {
+    fn unlock_after_execution(address_book: &mut AddressBook, mut lock_attempts: Vec<LockAttempt>) {
         for l in lock_attempts {
-            let newly_uncontended_while_queued = address_book.unlock(&l);
+            let newly_uncontended_while_queued = address_book.unlock(&mut l);
             if newly_uncontended_while_queued {
                 address_book.newly_uncontended_addresses.insert(*l.status.address());
             }

From f4a3cecf80bbeacbb61f85c17a9917000f3f1962 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 6 Aug 2022 22:46:31 +0900
Subject: [PATCH 0154/3199] save

---
 scheduler/src/lib.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index a0c9dc39239cd7..f103ca36f31c89 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -519,9 +519,9 @@ impl ScheduleStage {
     fn finalize_successful_lock_before_execution(
         address_book: &mut AddressBook,
         unique_weight: &UniqueWeight,
-        lock_attempts: &Vec<LockAttempt>,
+        lock_attempts: &mut Vec<LockAttempt>,
     ) {
-        for l in lock_attempts {
+        for mut l in lock_attempts {
             // ensure to remove remaining refs of this unique_weight
             address_book.forget_address_contention(&unique_weight, &mut l);
 

From 5e65269c0792db3618f7089fd88256be31dc39e0 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 6 Aug 2022 22:46:43 +0900
Subject: [PATCH 0155/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index f103ca36f31c89..2ce557bcd7709f 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -505,7 +505,7 @@ impl ScheduleStage {
             Self::finalize_successful_lock_before_execution(
                 address_book,
                 &unique_weight,
-                &populated_lock_attempts,
+                &mut populated_lock_attempts,
             );
             let task = queue_entry.remove();
             return Some((unique_weight, task, populated_lock_attempts));

From 6210f3fda7933d46a26f5de3d46b357e0f4c084d Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 6 Aug 2022 22:46:56 +0900
Subject: [PATCH 0156/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 2ce557bcd7709f..ac0eb72eef3fb8 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -549,7 +549,7 @@ impl ScheduleStage {
     }
 
     #[inline(never)]
-    fn unlock_after_execution(address_book: &mut AddressBook, mut lock_attempts: Vec<LockAttempt>) {
+    fn unlock_after_execution(address_book: &mut AddressBook, lock_attempts: Vec<LockAttempt>) {
         for l in lock_attempts {
             let newly_uncontended_while_queued = address_book.unlock(&mut l);
             if newly_uncontended_while_queued {

From 18731290933c38f6e8163089b4cbb95c44f7ed91 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 6 Aug 2022 22:47:05 +0900
Subject: [PATCH 0157/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index ac0eb72eef3fb8..cfd3ce2252ca0a 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -550,7 +550,7 @@ impl ScheduleStage {
 
     #[inline(never)]
     fn unlock_after_execution(address_book: &mut AddressBook, lock_attempts: Vec<LockAttempt>) {
-        for l in lock_attempts {
+        for mut l in lock_attempts {
             let newly_uncontended_while_queued = address_book.unlock(&mut l);
             if newly_uncontended_while_queued {
                 address_book.newly_uncontended_addresses.insert(*l.status.address());

From 660fbb55c65a601f39a54c1847c67838d86fca2e Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 6 Aug 2022 22:48:47 +0900
Subject: [PATCH 0158/3199] save

---
 scheduler/src/lib.rs | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index cfd3ce2252ca0a..dd553d5a4eed89 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -155,7 +155,9 @@ impl AddressBook {
 
         match self.book.entry(*status.address()) {
             AddressMapEntry::Vacant(book_entry) => {
-                book_entry.insert(Rc::new(Page::new(CurrentUsage::renew(*requested_usage))));
+                let page = Rc::new(Page::new(CurrentUsage::renew(*requested_usage)));
+                book_entry.insert(page);
+                *status = Rc::clone(page);
                 *is_success = true;
             }
             AddressMapEntry::Occupied(mut book_entry) => {

From 4acda72d88eeb254a4da3a3045e7e6a04c872da2 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 6 Aug 2022 22:48:54 +0900
Subject: [PATCH 0159/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index dd553d5a4eed89..8de1f4ad85f00d 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -157,7 +157,7 @@ impl AddressBook {
             AddressMapEntry::Vacant(book_entry) => {
                 let page = Rc::new(Page::new(CurrentUsage::renew(*requested_usage)));
                 book_entry.insert(page);
-                *status = Rc::clone(page);
+                *status = Rc::clone(&page);
                 *is_success = true;
             }
             AddressMapEntry::Occupied(mut book_entry) => {

From c5498080b26a0a45b104bd3f25b14069e4b32a5e Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 6 Aug 2022 22:49:43 +0900
Subject: [PATCH 0160/3199] save

---
 scheduler/src/lib.rs | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 8de1f4ad85f00d..d239b28ff6b0c5 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -153,11 +153,12 @@ impl AddressBook {
     ) {
         let LockAttempt {status, requested_usage, is_success} = attempt;
 
-        match self.book.entry(*status.address()) {
+        let address = status.address();
+        match self.book.entry(*address) {
             AddressMapEntry::Vacant(book_entry) => {
                 let page = Rc::new(Page::new(CurrentUsage::renew(*requested_usage)));
                 book_entry.insert(page);
-                *status = Rc::clone(&page);
+                *status = LockAttemptStatus::AfterLookup(*address, Rc::clone(&page));
                 *is_success = true;
             }
             AddressMapEntry::Occupied(mut book_entry) => {

From 222ab483c8259738dac6c11eca473da662523de2 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 6 Aug 2022 22:49:59 +0900
Subject: [PATCH 0161/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index d239b28ff6b0c5..2db12fcb68abff 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -157,9 +157,9 @@ impl AddressBook {
         match self.book.entry(*address) {
             AddressMapEntry::Vacant(book_entry) => {
                 let page = Rc::new(Page::new(CurrentUsage::renew(*requested_usage)));
-                book_entry.insert(page);
                 *status = LockAttemptStatus::AfterLookup(*address, Rc::clone(&page));
                 *is_success = true;
+                book_entry.insert(page);
             }
             AddressMapEntry::Occupied(mut book_entry) => {
                 let mut page = Rc::<Page>::get_mut(book_entry.get_mut()).unwrap();

From 1f15539a22684eb7d3a6ac077709285cebbc78e8 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 6 Aug 2022 22:50:26 +0900
Subject: [PATCH 0162/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 2db12fcb68abff..83d309616aeecd 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -162,7 +162,7 @@ impl AddressBook {
                 book_entry.insert(page);
             }
             AddressMapEntry::Occupied(mut book_entry) => {
-                let mut page = Rc::<Page>::get_mut(book_entry.get_mut()).unwrap();
+                let mut page = Rc::get_mut(book_entry.get_mut()).unwrap();
 
                 match page.current_usage {
                     CurrentUsage::Unused => {

From a759d637eacb49070a6bceb23b9361f5b2098c39 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 6 Aug 2022 22:51:02 +0900
Subject: [PATCH 0163/3199] save

---
 scheduler/src/lib.rs | 1 +
 1 file changed, 1 insertion(+)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 83d309616aeecd..7a90ec2fda4966 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -163,6 +163,7 @@ impl AddressBook {
             }
             AddressMapEntry::Occupied(mut book_entry) => {
                 let mut page = Rc::get_mut(book_entry.get_mut()).unwrap();
+                *status = LockAttemptStatus::AfterLookup(*address, Rc::clone(&page));
 
                 match page.current_usage {
                     CurrentUsage::Unused => {

From 9a7233f6b221621d702eed6a4137fd00c642cc20 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 6 Aug 2022 22:51:36 +0900
Subject: [PATCH 0164/3199] save

---
 scheduler/src/lib.rs | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 7a90ec2fda4966..de134a11577960 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -162,7 +162,8 @@ impl AddressBook {
                 book_entry.insert(page);
             }
             AddressMapEntry::Occupied(mut book_entry) => {
-                let mut page = Rc::get_mut(book_entry.get_mut()).unwrap();
+                let page = book_entry.get_mut();
+                let mut page = Rc::get_mut(page).unwrap();
                 *status = LockAttemptStatus::AfterLookup(*address, Rc::clone(&page));
 
                 match page.current_usage {

From 71ff270f79e97c698533c6f84af943e8bc009aa8 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 6 Aug 2022 22:51:42 +0900
Subject: [PATCH 0165/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index de134a11577960..0441084e35914b 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -163,8 +163,8 @@ impl AddressBook {
             }
             AddressMapEntry::Occupied(mut book_entry) => {
                 let page = book_entry.get_mut();
-                let mut page = Rc::get_mut(page).unwrap();
                 *status = LockAttemptStatus::AfterLookup(*address, Rc::clone(&page));
+                let mut page = Rc::get_mut(page).unwrap();
 
                 match page.current_usage {
                     CurrentUsage::Unused => {

From 64f5f08f020cfb611758f4716c976bbc243fe126 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 6 Aug 2022 22:52:09 +0900
Subject: [PATCH 0166/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 0441084e35914b..8106bcad4c856a 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -163,7 +163,7 @@ impl AddressBook {
             }
             AddressMapEntry::Occupied(mut book_entry) => {
                 let page = book_entry.get_mut();
-                *status = LockAttemptStatus::AfterLookup(*address, Rc::clone(&page));
+                *status = LockAttemptStatus::AfterLookup(*address, Rc::clone(page));
                 let mut page = Rc::get_mut(page).unwrap();
 
                 match page.current_usage {

From f82115847cf3c27b6b8e450e8754f4b3e070eec7 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 6 Aug 2022 22:52:33 +0900
Subject: [PATCH 0167/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 8106bcad4c856a..0441084e35914b 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -163,7 +163,7 @@ impl AddressBook {
             }
             AddressMapEntry::Occupied(mut book_entry) => {
                 let page = book_entry.get_mut();
-                *status = LockAttemptStatus::AfterLookup(*address, Rc::clone(page));
+                *status = LockAttemptStatus::AfterLookup(*address, Rc::clone(&page));
                 let mut page = Rc::get_mut(page).unwrap();
 
                 match page.current_usage {

From 90717b26ca4ee7cf1e12d56d45e7982c86e14976 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 6 Aug 2022 22:53:05 +0900
Subject: [PATCH 0168/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 0441084e35914b..dfd253cbb9cfc6 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -46,7 +46,7 @@ enum LockAttemptStatus {
 }
 
 impl LockAttemptStatus {
-    fn address(&self) -> &Pubkey {
+    fn address2(&self) -> &Pubkey {
         match self {
             LockAttemptStatus::BeforeLookup(pubkey) => &pubkey,
             LockAttemptStatus::AfterLookup(pubkey, _) => &pubkey,

From 44b38b45b4b2c2432b1f8eae80f49540013c1122 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 6 Aug 2022 22:53:15 +0900
Subject: [PATCH 0169/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index dfd253cbb9cfc6..671b525a0eec3b 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -153,7 +153,7 @@ impl AddressBook {
     ) {
         let LockAttempt {status, requested_usage, is_success} = attempt;
 
-        let address = status.address();
+        let address = status.address2();
         match self.book.entry(*address) {
             AddressMapEntry::Vacant(book_entry) => {
                 let page = Rc::new(Page::new(CurrentUsage::renew(*requested_usage)));

From 9a43bfbda0332e6bf7b48c69ee39902e661595ae Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 6 Aug 2022 22:53:52 +0900
Subject: [PATCH 0170/3199] save

---
 scheduler/src/lib.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 671b525a0eec3b..dacbd91f05d60d 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -531,7 +531,7 @@ impl ScheduleStage {
             address_book.forget_address_contention(&unique_weight, &mut l);
 
             // revert because now contended again
-            address_book.newly_uncontended_addresses.remove(&l.status.address());
+            address_book.newly_uncontended_addresses.remove(&l.status.address2());
         }
     }
 
@@ -546,7 +546,7 @@ impl ScheduleStage {
 
             // revert because now contended again
             if !from_runnable {
-                address_book.newly_uncontended_addresses.remove(&l.status.address());
+                address_book.newly_uncontended_addresses.remove(&l.status.address2());
             }
 
             // todo: mem::forget and panic in LockAttempt::drop()

From 1534b0dd73a74b6b6c42b53cddd2710c0faa7549 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 6 Aug 2022 22:54:43 +0900
Subject: [PATCH 0171/3199] save

---
 scheduler/src/lib.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index dacbd91f05d60d..0f6507f45d884f 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -140,7 +140,7 @@ type AddressMapEntry<'a, K, V> = std::collections::hash_map::Entry<'a, K, V>;
 #[derive(Default)]
 pub struct AddressBook {
     book: AddressMap,
-    newly_uncontended_addresses: AddressSet,
+    newly_uncontended_addresses: AddressMap,
 }
 
 impl AddressBook {
@@ -558,7 +558,7 @@ impl ScheduleStage {
         for mut l in lock_attempts {
             let newly_uncontended_while_queued = address_book.unlock(&mut l);
             if newly_uncontended_while_queued {
-                address_book.newly_uncontended_addresses.insert(*l.status.address());
+                address_book.newly_uncontended_addresses.insert(*l.status.address2(), Rc::clone(*l));
             }
 
             // todo: mem::forget and panic in LockAttempt::drop()

From be86046a2144ad7b23be7d1d50e54fe6b9fecded Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 6 Aug 2022 22:55:25 +0900
Subject: [PATCH 0172/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 0f6507f45d884f..cbd032125797da 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -409,7 +409,7 @@ impl ScheduleStage {
     fn get_weight_from_contended(address_book: &AddressBook) -> Option<UniqueWeight> {
         let mut heaviest_weight: Option<UniqueWeight> = None;
         //trace!("n u a len(): {}", address_book.newly_uncontended_addresses.len());
-        for address in address_book.newly_uncontended_addresses.iter() {
+        for address in address_book.newly_uncontended_addresses.values() {
             let newly_uncontended_unique_weights = Self::get_newly_u_u_w(address, &address_book);
             if let Some(&weight) = newly_uncontended_unique_weights.last() {
                 if let Some(current_heaviest_weight) = heaviest_weight {

From abcf17a08eb44257ed1a68290d4fc27890c1ee2f Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 6 Aug 2022 22:55:59 +0900
Subject: [PATCH 0173/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index cbd032125797da..e7b79920e027da 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -410,7 +410,7 @@ impl ScheduleStage {
         let mut heaviest_weight: Option<UniqueWeight> = None;
         //trace!("n u a len(): {}", address_book.newly_uncontended_addresses.len());
         for address in address_book.newly_uncontended_addresses.values() {
-            let newly_uncontended_unique_weights = Self::get_newly_u_u_w(address, &address_book);
+            let newly_uncontended_unique_weights = address.page().contended_unique_weights;
             if let Some(&weight) = newly_uncontended_unique_weights.last() {
                 if let Some(current_heaviest_weight) = heaviest_weight {
                     if weight > current_heaviest_weight {

From 338e39b40e3c134d4398d6558fd65760c34d333d Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 6 Aug 2022 22:56:09 +0900
Subject: [PATCH 0174/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index e7b79920e027da..7f96c486f4986c 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -410,7 +410,7 @@ impl ScheduleStage {
         let mut heaviest_weight: Option<UniqueWeight> = None;
         //trace!("n u a len(): {}", address_book.newly_uncontended_addresses.len());
         for address in address_book.newly_uncontended_addresses.values() {
-            let newly_uncontended_unique_weights = address.page().contended_unique_weights;
+            let newly_uncontended_unique_weights = address.status.page().contended_unique_weights;
             if let Some(&weight) = newly_uncontended_unique_weights.last() {
                 if let Some(current_heaviest_weight) = heaviest_weight {
                     if weight > current_heaviest_weight {

From feb5d41e3c788093c3c088392b37d537af7f13df Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 6 Aug 2022 22:57:07 +0900
Subject: [PATCH 0175/3199] save

---
 scheduler/src/lib.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 7f96c486f4986c..0325b5d4c424e2 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -409,8 +409,8 @@ impl ScheduleStage {
     fn get_weight_from_contended(address_book: &AddressBook) -> Option<UniqueWeight> {
         let mut heaviest_weight: Option<UniqueWeight> = None;
         //trace!("n u a len(): {}", address_book.newly_uncontended_addresses.len());
-        for address in address_book.newly_uncontended_addresses.values() {
-            let newly_uncontended_unique_weights = address.status.page().contended_unique_weights;
+        for page in address_book.newly_uncontended_addresses.values() {
+            let newly_uncontended_unique_weights = page.contended_unique_weights;
             if let Some(&weight) = newly_uncontended_unique_weights.last() {
                 if let Some(current_heaviest_weight) = heaviest_weight {
                     if weight > current_heaviest_weight {

From 85977f257f752d8ef2c07105f8edd3cedd73b692 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 6 Aug 2022 22:57:24 +0900
Subject: [PATCH 0176/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 0325b5d4c424e2..f9f1dcf39ebdd6 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -558,7 +558,7 @@ impl ScheduleStage {
         for mut l in lock_attempts {
             let newly_uncontended_while_queued = address_book.unlock(&mut l);
             if newly_uncontended_while_queued {
-                address_book.newly_uncontended_addresses.insert(*l.status.address2(), Rc::clone(*l));
+                address_book.newly_uncontended_addresses.insert(l.status.address2(), Rc::clone(*l));
             }
 
             // todo: mem::forget and panic in LockAttempt::drop()

From 1b6ab7a76c71352421e1c63cf2a7a172729bb988 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 6 Aug 2022 22:57:33 +0900
Subject: [PATCH 0177/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index f9f1dcf39ebdd6..97204a23335a72 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -558,7 +558,7 @@ impl ScheduleStage {
         for mut l in lock_attempts {
             let newly_uncontended_while_queued = address_book.unlock(&mut l);
             if newly_uncontended_while_queued {
-                address_book.newly_uncontended_addresses.insert(l.status.address2(), Rc::clone(*l));
+                address_book.newly_uncontended_addresses.insert(*(l.status.address2()), Rc::clone(*l));
             }
 
             // todo: mem::forget and panic in LockAttempt::drop()

From 1cd5383c6ec95c52419273cd25d8b439ae758c9b Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 6 Aug 2022 22:57:42 +0900
Subject: [PATCH 0178/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 97204a23335a72..0461ba62eee70b 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -558,7 +558,7 @@ impl ScheduleStage {
         for mut l in lock_attempts {
             let newly_uncontended_while_queued = address_book.unlock(&mut l);
             if newly_uncontended_while_queued {
-                address_book.newly_uncontended_addresses.insert(*(l.status.address2()), Rc::clone(*l));
+                address_book.newly_uncontended_addresses.insert(*(l.status.address2()), Rc::clone(l));
             }
 
             // todo: mem::forget and panic in LockAttempt::drop()

From 454c23d65107bec3e9b4647438368d104a7e1ad9 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 6 Aug 2022 22:58:00 +0900
Subject: [PATCH 0179/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 0461ba62eee70b..0e4e70391eb1d4 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -558,7 +558,7 @@ impl ScheduleStage {
         for mut l in lock_attempts {
             let newly_uncontended_while_queued = address_book.unlock(&mut l);
             if newly_uncontended_while_queued {
-                address_book.newly_uncontended_addresses.insert(*(l.status.address2()), Rc::clone(l));
+                address_book.newly_uncontended_addresses.insert(*(l.status.address2()), Rc::clone(l.status.page()));
             }
 
             // todo: mem::forget and panic in LockAttempt::drop()

From 782d3c55bb8153a6a1d2a85c348552b5a57cad40 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 6 Aug 2022 22:58:47 +0900
Subject: [PATCH 0180/3199] save

---
 scheduler/src/lib.rs | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 0e4e70391eb1d4..c2a9fe80595eee 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -53,6 +53,13 @@ impl LockAttemptStatus {
         }
     }
 
+    fn page_rc(&mut self) -> &Rc<Page> {
+        match self {
+            LockAttemptStatus::BeforeLookup(_) => unreachable!(),
+            LockAttemptStatus::AfterLookup(_address, page) => page,
+        }
+    }
+
     fn page(&mut self) -> &mut Page {
         match self {
             LockAttemptStatus::BeforeLookup(_) => unreachable!(),

From d5ca3f0b8e615bdf1d11ba6cd8a8993316fa6e81 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 6 Aug 2022 22:58:56 +0900
Subject: [PATCH 0181/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index c2a9fe80595eee..1ac4b505246f3e 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -565,7 +565,7 @@ impl ScheduleStage {
         for mut l in lock_attempts {
             let newly_uncontended_while_queued = address_book.unlock(&mut l);
             if newly_uncontended_while_queued {
-                address_book.newly_uncontended_addresses.insert(*(l.status.address2()), Rc::clone(l.status.page()));
+                address_book.newly_uncontended_addresses.insert(*(l.status.address2()), Rc::clone(l.status.page_rc()));
             }
 
             // todo: mem::forget and panic in LockAttempt::drop()

From 585a474de19aa5c2832ff159460ec84586aed826 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 6 Aug 2022 22:59:05 +0900
Subject: [PATCH 0182/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 1ac4b505246f3e..51f480160beb3e 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -417,7 +417,7 @@ impl ScheduleStage {
         let mut heaviest_weight: Option<UniqueWeight> = None;
         //trace!("n u a len(): {}", address_book.newly_uncontended_addresses.len());
         for page in address_book.newly_uncontended_addresses.values() {
-            let newly_uncontended_unique_weights = page.contended_unique_weights;
+            let newly_uncontended_unique_weights = &page.contended_unique_weights;
             if let Some(&weight) = newly_uncontended_unique_weights.last() {
                 if let Some(current_heaviest_weight) = heaviest_weight {
                     if weight > current_heaviest_weight {

From fb833e059ebe4a9ae3fbd47cc69ae8465d444394 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 6 Aug 2022 23:01:34 +0900
Subject: [PATCH 0183/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 51f480160beb3e..94cb7b0c857fb7 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -16,7 +16,7 @@ use {
     },
 };
 
-use std::rc::Rc;
+type MyRc = std::rc::Rc;
 
 #[derive(Debug)]
 pub struct ExecutionEnvironment {

From 9851732be7e909d03e3df7e56061ca84426796b3 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 6 Aug 2022 23:01:48 +0900
Subject: [PATCH 0184/3199] save

---
 scheduler/src/lib.rs | 18 +++++++++---------
 1 file changed, 9 insertions(+), 9 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 94cb7b0c857fb7..fab458db17ed83 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -42,7 +42,7 @@ impl ExecutionEnvironment {
 #[derive(Clone, Debug)]
 enum LockAttemptStatus {
     BeforeLookup(Pubkey),
-    AfterLookup(Pubkey, Rc<Page>),
+    AfterLookup(Pubkey, MyRc<Page>),
 }
 
 impl LockAttemptStatus {
@@ -53,7 +53,7 @@ impl LockAttemptStatus {
         }
     }
 
-    fn page_rc(&mut self) -> &Rc<Page> {
+    fn page_rc(&mut self) -> &MyRc<Page> {
         match self {
             LockAttemptStatus::BeforeLookup(_) => unreachable!(),
             LockAttemptStatus::AfterLookup(_address, page) => page,
@@ -63,7 +63,7 @@ impl LockAttemptStatus {
     fn page(&mut self) -> &mut Page {
         match self {
             LockAttemptStatus::BeforeLookup(_) => unreachable!(),
-            LockAttemptStatus::AfterLookup(_address, page) => Rc::get_mut(page).unwrap(),
+            LockAttemptStatus::AfterLookup(_address, page) => MyRc::get_mut(page).unwrap(),
         }
     }
 }
@@ -139,7 +139,7 @@ impl Page {
     }
 }
 
-type AddressMap = std::collections::HashMap<Pubkey, std::rc::Rc<Page>>;
+type AddressMap = std::collections::HashMap<Pubkey, std::rc::MyRc<Page>>;
 type AddressSet = std::collections::HashSet<Pubkey>;
 type AddressMapEntry<'a, K, V> = std::collections::hash_map::Entry<'a, K, V>;
 
@@ -163,15 +163,15 @@ impl AddressBook {
         let address = status.address2();
         match self.book.entry(*address) {
             AddressMapEntry::Vacant(book_entry) => {
-                let page = Rc::new(Page::new(CurrentUsage::renew(*requested_usage)));
-                *status = LockAttemptStatus::AfterLookup(*address, Rc::clone(&page));
+                let page = MyRc::new(Page::new(CurrentUsage::renew(*requested_usage)));
+                *status = LockAttemptStatus::AfterLookup(*address, MyRc::clone(&page));
                 *is_success = true;
                 book_entry.insert(page);
             }
             AddressMapEntry::Occupied(mut book_entry) => {
                 let page = book_entry.get_mut();
-                *status = LockAttemptStatus::AfterLookup(*address, Rc::clone(&page));
-                let mut page = Rc::get_mut(page).unwrap();
+                *status = LockAttemptStatus::AfterLookup(*address, MyRc::clone(&page));
+                let mut page = MyRc::get_mut(page).unwrap();
 
                 match page.current_usage {
                     CurrentUsage::Unused => {
@@ -565,7 +565,7 @@ impl ScheduleStage {
         for mut l in lock_attempts {
             let newly_uncontended_while_queued = address_book.unlock(&mut l);
             if newly_uncontended_while_queued {
-                address_book.newly_uncontended_addresses.insert(*(l.status.address2()), Rc::clone(l.status.page_rc()));
+                address_book.newly_uncontended_addresses.insert(*(l.status.address2()), MyRc::clone(l.status.page_rc()));
             }
 
             // todo: mem::forget and panic in LockAttempt::drop()

From aa9a4ee9f8a1acff75896f4f39212c6c1e1d4bf4 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 6 Aug 2022 23:02:04 +0900
Subject: [PATCH 0185/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index fab458db17ed83..69fa76fd72ef00 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -16,7 +16,7 @@ use {
     },
 };
 
-type MyRc = std::rc::Rc;
+type MyRc<T> = std::rc::Rc<T>;
 
 #[derive(Debug)]
 pub struct ExecutionEnvironment {

From 81b7690a0ce6ad094218f71861ca0ee68a139422 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 6 Aug 2022 23:03:11 +0900
Subject: [PATCH 0186/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 69fa76fd72ef00..26acdf52b2e2dc 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -139,7 +139,7 @@ impl Page {
     }
 }
 
-type AddressMap = std::collections::HashMap<Pubkey, std::rc::MyRc<Page>>;
+type AddressMap = std::collections::HashMap<Pubkey, MyRc<Page>>;
 type AddressSet = std::collections::HashSet<Pubkey>;
 type AddressMapEntry<'a, K, V> = std::collections::hash_map::Entry<'a, K, V>;
 

From dee7d8986f0157981ecaaea47314e5058907e63a Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 6 Aug 2022 23:03:25 +0900
Subject: [PATCH 0187/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 26acdf52b2e2dc..b8552cdb85070b 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -16,7 +16,7 @@ use {
     },
 };
 
-type MyRc<T> = std::rc::Rc<T>;
+type MyRc<T> = std::rc::Arc<T>;
 
 #[derive(Debug)]
 pub struct ExecutionEnvironment {

From a00cb718f33d83e1803d161bf2dd30d5b3fcae6c Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 6 Aug 2022 23:03:38 +0900
Subject: [PATCH 0188/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index b8552cdb85070b..611d4e11b360dd 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -16,7 +16,7 @@ use {
     },
 };
 
-type MyRc<T> = std::rc::Arc<T>;
+type MyRc<T> = std::sync::Arc<T>;
 
 #[derive(Debug)]
 pub struct ExecutionEnvironment {

From c3a09e9b70a3b40d22d479791f40d6d870bc9a2b Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 6 Aug 2022 23:06:16 +0900
Subject: [PATCH 0189/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 611d4e11b360dd..7ced42cd1c6075 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -171,7 +171,7 @@ impl AddressBook {
             AddressMapEntry::Occupied(mut book_entry) => {
                 let page = book_entry.get_mut();
                 *status = LockAttemptStatus::AfterLookup(*address, MyRc::clone(&page));
-                let mut page = MyRc::get_mut(page).unwrap();
+                let mut page = MyRc::mut(page).unwrap();
 
                 match page.current_usage {
                     CurrentUsage::Unused => {

From 07344b8d30352fc753ab477eeec250d8fcf101a9 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 6 Aug 2022 23:06:30 +0900
Subject: [PATCH 0190/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 7ced42cd1c6075..e1ca2a1e7902ee 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -171,7 +171,7 @@ impl AddressBook {
             AddressMapEntry::Occupied(mut book_entry) => {
                 let page = book_entry.get_mut();
                 *status = LockAttemptStatus::AfterLookup(*address, MyRc::clone(&page));
-                let mut page = MyRc::mut(page).unwrap();
+                let mut page = MyRc::ref_mut(page).unwrap();
 
                 match page.current_usage {
                     CurrentUsage::Unused => {

From 8c7624ba1bf4be56817a35885fd37b67b5117941 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 6 Aug 2022 23:09:21 +0900
Subject: [PATCH 0191/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index e1ca2a1e7902ee..1e9f0449ea20c3 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -171,7 +171,7 @@ impl AddressBook {
             AddressMapEntry::Occupied(mut book_entry) => {
                 let page = book_entry.get_mut();
                 *status = LockAttemptStatus::AfterLookup(*address, MyRc::clone(&page));
-                let mut page = MyRc::ref_mut(page).unwrap();
+                let mut page = MyRc::get_mut_unchecked(page).unwrap();
 
                 match page.current_usage {
                     CurrentUsage::Unused => {

From 8cf951c3dacff24f51af6d9c4869fe79c41a1e3b Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 6 Aug 2022 23:09:32 +0900
Subject: [PATCH 0192/3199] save

---
 scheduler/src/lib.rs | 1 +
 1 file changed, 1 insertion(+)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 1e9f0449ea20c3..f9a786f145bf2d 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1,4 +1,5 @@
 #![feature(map_first_last)]
+#![feature(get_mut_unchecked)]
 
 use {
     atomic_enum::atomic_enum,

From da851e2cbcb4df7bc6a85109ca9df078ad0266ab Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 6 Aug 2022 23:09:45 +0900
Subject: [PATCH 0193/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index f9a786f145bf2d..502a1f211c6436 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -172,7 +172,7 @@ impl AddressBook {
             AddressMapEntry::Occupied(mut book_entry) => {
                 let page = book_entry.get_mut();
                 *status = LockAttemptStatus::AfterLookup(*address, MyRc::clone(&page));
-                let mut page = MyRc::get_mut_unchecked(page).unwrap();
+                let mut page = MyRc::get_mut_unchecked(page);
 
                 match page.current_usage {
                     CurrentUsage::Unused => {

From 6352c530781be30a9863ec16c3f8ffa6ea793948 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 6 Aug 2022 23:09:56 +0900
Subject: [PATCH 0194/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 502a1f211c6436..378e2c16ba298b 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -172,7 +172,7 @@ impl AddressBook {
             AddressMapEntry::Occupied(mut book_entry) => {
                 let page = book_entry.get_mut();
                 *status = LockAttemptStatus::AfterLookup(*address, MyRc::clone(&page));
-                let mut page = MyRc::get_mut_unchecked(page);
+                let mut page = unsafe MyRc::get_mut_unchecked(page);
 
                 match page.current_usage {
                     CurrentUsage::Unused => {

From 4ae3cdf03fc69e79b8ef59aeccdb490a927b2ae0 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 6 Aug 2022 23:10:04 +0900
Subject: [PATCH 0195/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 378e2c16ba298b..bb93859408832c 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -172,7 +172,7 @@ impl AddressBook {
             AddressMapEntry::Occupied(mut book_entry) => {
                 let page = book_entry.get_mut();
                 *status = LockAttemptStatus::AfterLookup(*address, MyRc::clone(&page));
-                let mut page = unsafe MyRc::get_mut_unchecked(page);
+                let mut page = unsafe { MyRc::get_mut_unchecked(page) };
 
                 match page.current_usage {
                     CurrentUsage::Unused => {

From 5e123b68b0fc8dc9ab2252e234d3435f65b2d621 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 6 Aug 2022 23:10:38 +0900
Subject: [PATCH 0196/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index bb93859408832c..00383743a28190 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -64,7 +64,7 @@ impl LockAttemptStatus {
     fn page(&mut self) -> &mut Page {
         match self {
             LockAttemptStatus::BeforeLookup(_) => unreachable!(),
-            LockAttemptStatus::AfterLookup(_address, page) => MyRc::get_mut(page).unwrap(),
+            LockAttemptStatus::AfterLookup(_address, page) => MyRc::get_mut_unchecked(page).unwrap(),
         }
     }
 }

From 2df52b0f2cb5ca3b79d117d13a426733fb9a6a50 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 6 Aug 2022 23:10:50 +0900
Subject: [PATCH 0197/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 00383743a28190..2f6c5c148c737d 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -64,7 +64,7 @@ impl LockAttemptStatus {
     fn page(&mut self) -> &mut Page {
         match self {
             LockAttemptStatus::BeforeLookup(_) => unreachable!(),
-            LockAttemptStatus::AfterLookup(_address, page) => MyRc::get_mut_unchecked(page).unwrap(),
+            LockAttemptStatus::AfterLookup(_address, page) => MyRc::get_mut_unchecked(page),
         }
     }
 }

From c538ab369598ac932b60267414b569f404497144 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 6 Aug 2022 23:11:01 +0900
Subject: [PATCH 0198/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 2f6c5c148c737d..6a48992b144e2e 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -64,7 +64,7 @@ impl LockAttemptStatus {
     fn page(&mut self) -> &mut Page {
         match self {
             LockAttemptStatus::BeforeLookup(_) => unreachable!(),
-            LockAttemptStatus::AfterLookup(_address, page) => MyRc::get_mut_unchecked(page),
+            LockAttemptStatus::AfterLookup(_address, page) => unsafe { MyRc::get_mut_unchecked(page) },
         }
     }
 }

From 41c4f6f9410107b0c6517f16b499a3f055510c70 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 7 Aug 2022 21:38:42 +0900
Subject: [PATCH 0199/3199] save

---
 scheduler/Cargo.toml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/scheduler/Cargo.toml b/scheduler/Cargo.toml
index 308dcef5aaf11a..3935355d558249 100644
--- a/scheduler/Cargo.toml
+++ b/scheduler/Cargo.toml
@@ -12,6 +12,7 @@ publish = true
 
 [dependencies]
 atomic_enum = "0.1.1"
+by_address = "1.0.4"
 crossbeam-channel = "0.5.5"
 log = "0.4.17"
 solana-entry = { path = "../entry", version = "=1.11.5" }

From 93295765b7c3097c82f55b9b416e4163b7859c85 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 7 Aug 2022 21:40:35 +0900
Subject: [PATCH 0200/3199] save

---
 Cargo.lock           | 7 +++++++
 scheduler/src/lib.rs | 4 ++--
 2 files changed, 9 insertions(+), 2 deletions(-)

diff --git a/Cargo.lock b/Cargo.lock
index 46428aa41fe49a..1626ee9ab5d8d3 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -581,6 +581,12 @@ dependencies = [
  "serde",
 ]
 
+[[package]]
+name = "by_address"
+version = "1.0.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e245704f60eb4eb45810d65cf14eb54d2eb50a6f3715fe2d7cd01ee905c2944f"
+
 [[package]]
 name = "byte-tools"
 version = "0.3.1"
@@ -6067,6 +6073,7 @@ name = "solana-scheduler"
 version = "1.11.5"
 dependencies = [
  "atomic_enum",
+ "by_address",
  "crossbeam-channel",
  "log",
  "rand 0.7.3",
diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 6a48992b144e2e..e4494ac20b365d 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -141,14 +141,14 @@ impl Page {
 }
 
 type AddressMap = std::collections::HashMap<Pubkey, MyRc<Page>>;
-type AddressSet = std::collections::HashSet<Pubkey>;
+type AddressSet = std::collections::HashSet<ByAddress<MyRc<Page>>>;
 type AddressMapEntry<'a, K, V> = std::collections::hash_map::Entry<'a, K, V>;
 
 // needs ttl mechanism and prune
 #[derive(Default)]
 pub struct AddressBook {
     book: AddressMap,
-    newly_uncontended_addresses: AddressMap,
+    newly_uncontended_addresses: AddressSet,
 }
 
 impl AddressBook {

From cd86a1f6a9ed0c0cd289c9b9f722984c5468c495 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 7 Aug 2022 21:41:00 +0900
Subject: [PATCH 0201/3199] save

---
 scheduler/src/lib.rs | 1 +
 1 file changed, 1 insertion(+)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index e4494ac20b365d..c99f4229133528 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -141,6 +141,7 @@ impl Page {
 }
 
 type AddressMap = std::collections::HashMap<Pubkey, MyRc<Page>>;
+use by_address::ByAddress;
 type AddressSet = std::collections::HashSet<ByAddress<MyRc<Page>>>;
 type AddressMapEntry<'a, K, V> = std::collections::hash_map::Entry<'a, K, V>;
 

From f460e963d9dd0c064df984e4512af33b3f60a2fa Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 7 Aug 2022 21:41:15 +0900
Subject: [PATCH 0202/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index c99f4229133528..066f0707f58b48 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -418,7 +418,7 @@ impl ScheduleStage {
     fn get_weight_from_contended(address_book: &AddressBook) -> Option<UniqueWeight> {
         let mut heaviest_weight: Option<UniqueWeight> = None;
         //trace!("n u a len(): {}", address_book.newly_uncontended_addresses.len());
-        for page in address_book.newly_uncontended_addresses.values() {
+        for page in address_book.newly_uncontended_addresses.iter() {
             let newly_uncontended_unique_weights = &page.contended_unique_weights;
             if let Some(&weight) = newly_uncontended_unique_weights.last() {
                 if let Some(current_heaviest_weight) = heaviest_weight {

From 740d11c14fe73ac1e368e146726c5b1cbac670d6 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 7 Aug 2022 21:41:48 +0900
Subject: [PATCH 0203/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 066f0707f58b48..a4a0b70bae40d4 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -540,7 +540,7 @@ impl ScheduleStage {
             address_book.forget_address_contention(&unique_weight, &mut l);
 
             // revert because now contended again
-            address_book.newly_uncontended_addresses.remove(&l.status.address2());
+            address_book.newly_uncontended_addresses.remove(&l.status.page_rc());
         }
     }
 

From 152aadb578f49772f40145652f2a9bb016a58114 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 7 Aug 2022 21:43:06 +0900
Subject: [PATCH 0204/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index a4a0b70bae40d4..e1c2a1f8a7ec4f 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -540,7 +540,7 @@ impl ScheduleStage {
             address_book.forget_address_contention(&unique_weight, &mut l);
 
             // revert because now contended again
-            address_book.newly_uncontended_addresses.remove(&l.status.page_rc());
+            address_book.newly_uncontended_addresses.remove(&ByAddress(l.status.page_rc()));
         }
     }
 

From f4da0c53ccedad5ec85e4715957d8d191db521c1 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 7 Aug 2022 21:43:24 +0900
Subject: [PATCH 0205/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index e1c2a1f8a7ec4f..2341f33b52454f 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -540,7 +540,7 @@ impl ScheduleStage {
             address_book.forget_address_contention(&unique_weight, &mut l);
 
             // revert because now contended again
-            address_book.newly_uncontended_addresses.remove(&ByAddress(l.status.page_rc()));
+            address_book.newly_uncontended_addresses.remove(&ByAddress(*l.status.page_rc()));
         }
     }
 

From 0d3ea79c9d755207501d66087b30bee8110b15cf Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 7 Aug 2022 21:43:44 +0900
Subject: [PATCH 0206/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 2341f33b52454f..29abd7cb3724ea 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -555,7 +555,7 @@ impl ScheduleStage {
 
             // revert because now contended again
             if !from_runnable {
-                address_book.newly_uncontended_addresses.remove(&l.status.address2());
+                address_book.newly_uncontended_addresses.remove(&ByAddress(*l.status.page_rc()));
             }
 
             // todo: mem::forget and panic in LockAttempt::drop()

From f85f24f720f18c3ec22710d7f90546e09c04d8e8 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 7 Aug 2022 21:44:13 +0900
Subject: [PATCH 0207/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 29abd7cb3724ea..d0505512482c5b 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -567,7 +567,7 @@ impl ScheduleStage {
         for mut l in lock_attempts {
             let newly_uncontended_while_queued = address_book.unlock(&mut l);
             if newly_uncontended_while_queued {
-                address_book.newly_uncontended_addresses.insert(*(l.status.address2()), MyRc::clone(l.status.page_rc()));
+                address_book.newly_uncontended_addresses.insert(ByAddress(MyRc::clone(l.status.page_rc())));
             }
 
             // todo: mem::forget and panic in LockAttempt::drop()

From dc0f1b2653b6715a47095b79fddf2d05965bd8cc Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 7 Aug 2022 21:46:14 +0900
Subject: [PATCH 0208/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index d0505512482c5b..255366c19cdec8 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -17,7 +17,7 @@ use {
     },
 };
 
-type MyRc<T> = std::sync::Arc<T>;
+type MyRc<T> = ByAddress<std::sync::Arc<T>>;
 
 #[derive(Debug)]
 pub struct ExecutionEnvironment {

From 0a035034850594cebae113469ae82934455e87a6 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 7 Aug 2022 21:48:01 +0900
Subject: [PATCH 0209/3199] save

---
 scheduler/src/lib.rs | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 255366c19cdec8..813b921d2d7c61 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -17,7 +17,8 @@ use {
     },
 };
 
-type MyRc<T> = ByAddress<std::sync::Arc<T>>;
+type MyRcInner<T> = std::sync::Arc<T>;
+type MyRc<T> = ByAddress<MyRcInner>;
 
 #[derive(Debug)]
 pub struct ExecutionEnvironment {
@@ -64,7 +65,7 @@ impl LockAttemptStatus {
     fn page(&mut self) -> &mut Page {
         match self {
             LockAttemptStatus::BeforeLookup(_) => unreachable!(),
-            LockAttemptStatus::AfterLookup(_address, page) => unsafe { MyRc::get_mut_unchecked(page) },
+            LockAttemptStatus::AfterLookup(_address, page) => unsafe { MyRcInner::get_mut_unchecked(page) },
         }
     }
 }

From dcc1d270b45bd75d181e945966e54922111c46cd Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 7 Aug 2022 21:48:19 +0900
Subject: [PATCH 0210/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 813b921d2d7c61..6ee812dc55c8d5 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -18,7 +18,7 @@ use {
 };
 
 type MyRcInner<T> = std::sync::Arc<T>;
-type MyRc<T> = ByAddress<MyRcInner>;
+type MyRc<T> = ByAddress<MyRcInner<T>>;
 
 #[derive(Debug)]
 pub struct ExecutionEnvironment {

From 8699bdc92fa1e3984c32ec0241ea9651a0065be4 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 7 Aug 2022 21:48:31 +0900
Subject: [PATCH 0211/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 6ee812dc55c8d5..f926720b6eb4c5 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -174,7 +174,7 @@ impl AddressBook {
             AddressMapEntry::Occupied(mut book_entry) => {
                 let page = book_entry.get_mut();
                 *status = LockAttemptStatus::AfterLookup(*address, MyRc::clone(&page));
-                let mut page = unsafe { MyRc::get_mut_unchecked(page) };
+                let mut page = unsafe { MyRcInner::get_mut_unchecked(page) };
 
                 match page.current_usage {
                     CurrentUsage::Unused => {

From ae66e127e3da60c4cdb48be5a1948f9c9e585e60 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 7 Aug 2022 21:49:05 +0900
Subject: [PATCH 0212/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index f926720b6eb4c5..a97146f453787b 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -166,7 +166,7 @@ impl AddressBook {
         let address = status.address2();
         match self.book.entry(*address) {
             AddressMapEntry::Vacant(book_entry) => {
-                let page = MyRc::new(Page::new(CurrentUsage::renew(*requested_usage)));
+                let page = MyRc(MyRcInner::new(Page::new(CurrentUsage::renew(*requested_usage))));
                 *status = LockAttemptStatus::AfterLookup(*address, MyRc::clone(&page));
                 *is_success = true;
                 book_entry.insert(page);

From 382182447ae696278b12160dc43251dadbae9279 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 7 Aug 2022 21:49:21 +0900
Subject: [PATCH 0213/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index a97146f453787b..92bd243786d57f 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -166,7 +166,7 @@ impl AddressBook {
         let address = status.address2();
         match self.book.entry(*address) {
             AddressMapEntry::Vacant(book_entry) => {
-                let page = MyRc(MyRcInner::new(Page::new(CurrentUsage::renew(*requested_usage))));
+                let page = ByAddress(MyRcInner::new(Page::new(CurrentUsage::renew(*requested_usage))));
                 *status = LockAttemptStatus::AfterLookup(*address, MyRc::clone(&page));
                 *is_success = true;
                 book_entry.insert(page);

From e4b276d4b8014592005fabb437c0b71ddba4f7ee Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 7 Aug 2022 21:50:13 +0900
Subject: [PATCH 0214/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 92bd243786d57f..a81af06a72966e 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -568,7 +568,7 @@ impl ScheduleStage {
         for mut l in lock_attempts {
             let newly_uncontended_while_queued = address_book.unlock(&mut l);
             if newly_uncontended_while_queued {
-                address_book.newly_uncontended_addresses.insert(ByAddress(MyRc::clone(l.status.page_rc())));
+                address_book.newly_uncontended_addresses.insert(l.status.page_rc());
             }
 
             // todo: mem::forget and panic in LockAttempt::drop()

From 8a5fa46bcb999dbb76a0c3dc0459cedca730113b Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 7 Aug 2022 21:50:25 +0900
Subject: [PATCH 0215/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index a81af06a72966e..6a964bdad7498b 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -568,7 +568,7 @@ impl ScheduleStage {
         for mut l in lock_attempts {
             let newly_uncontended_while_queued = address_book.unlock(&mut l);
             if newly_uncontended_while_queued {
-                address_book.newly_uncontended_addresses.insert(l.status.page_rc());
+                address_book.newly_uncontended_addresses.insert(&l.status.page_rc());
             }
 
             // todo: mem::forget and panic in LockAttempt::drop()

From b584a02e1d93de3114e3555add74a6fd29185bcb Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 7 Aug 2022 21:50:38 +0900
Subject: [PATCH 0216/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 6a964bdad7498b..5a96402244f863 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -568,7 +568,7 @@ impl ScheduleStage {
         for mut l in lock_attempts {
             let newly_uncontended_while_queued = address_book.unlock(&mut l);
             if newly_uncontended_while_queued {
-                address_book.newly_uncontended_addresses.insert(&l.status.page_rc());
+                address_book.newly_uncontended_addresses.insert(*l.status.page_rc());
             }
 
             // todo: mem::forget and panic in LockAttempt::drop()

From 8aa62b46cd3b757aede5167538f847bbf6800b41 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 7 Aug 2022 21:50:51 +0900
Subject: [PATCH 0217/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 5a96402244f863..58b2c57da52df0 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -568,7 +568,7 @@ impl ScheduleStage {
         for mut l in lock_attempts {
             let newly_uncontended_while_queued = address_book.unlock(&mut l);
             if newly_uncontended_while_queued {
-                address_book.newly_uncontended_addresses.insert(*l.status.page_rc());
+                address_book.newly_uncontended_addresses.insert(l.status.page_rc().clone());
             }
 
             // todo: mem::forget and panic in LockAttempt::drop()

From 0d2f552e1d29b029c587800c43e6b9f566071664 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 7 Aug 2022 21:51:06 +0900
Subject: [PATCH 0218/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 58b2c57da52df0..b28b87a79cb547 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -568,7 +568,7 @@ impl ScheduleStage {
         for mut l in lock_attempts {
             let newly_uncontended_while_queued = address_book.unlock(&mut l);
             if newly_uncontended_while_queued {
-                address_book.newly_uncontended_addresses.insert(l.status.page_rc().clone());
+                address_book.newly_uncontended_addresses.insert(ByAddress(l.status.page_rc().clone()));
             }
 
             // todo: mem::forget and panic in LockAttempt::drop()

From 7b3ace5f7b972a1932acbcf9b82d9de642a68995 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 7 Aug 2022 21:51:21 +0900
Subject: [PATCH 0219/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index b28b87a79cb547..f14783c7ce9130 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -556,7 +556,7 @@ impl ScheduleStage {
 
             // revert because now contended again
             if !from_runnable {
-                address_book.newly_uncontended_addresses.remove(&ByAddress(*l.status.page_rc()));
+                address_book.newly_uncontended_addresses.remove(&l.status.page_rc());
             }
 
             // todo: mem::forget and panic in LockAttempt::drop()

From 94518f0b35437822623cb237e6a2a5215f43da1f Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 7 Aug 2022 21:51:50 +0900
Subject: [PATCH 0220/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index f14783c7ce9130..3880f8b817efe5 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -143,7 +143,7 @@ impl Page {
 
 type AddressMap = std::collections::HashMap<Pubkey, MyRc<Page>>;
 use by_address::ByAddress;
-type AddressSet = std::collections::HashSet<ByAddress<MyRc<Page>>>;
+type AddressSet = std::collections::HashSet<MyRc<Page>>;
 type AddressMapEntry<'a, K, V> = std::collections::hash_map::Entry<'a, K, V>;
 
 // needs ttl mechanism and prune

From 58f84753d5f929e42553f08af6f4d064e1616655 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 7 Aug 2022 21:52:11 +0900
Subject: [PATCH 0221/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 3880f8b817efe5..cd88dfa3a2bd3b 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -568,7 +568,7 @@ impl ScheduleStage {
         for mut l in lock_attempts {
             let newly_uncontended_while_queued = address_book.unlock(&mut l);
             if newly_uncontended_while_queued {
-                address_book.newly_uncontended_addresses.insert(ByAddress(l.status.page_rc().clone()));
+                address_book.newly_uncontended_addresses.insert(*l.status.page_rc());
             }
 
             // todo: mem::forget and panic in LockAttempt::drop()

From 36bebd83b5acf19d150b1923271489c7d0954e2c Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 7 Aug 2022 21:52:22 +0900
Subject: [PATCH 0222/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index cd88dfa3a2bd3b..fe8f973c7ceb15 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -541,7 +541,7 @@ impl ScheduleStage {
             address_book.forget_address_contention(&unique_weight, &mut l);
 
             // revert because now contended again
-            address_book.newly_uncontended_addresses.remove(&ByAddress(*l.status.page_rc()));
+            address_book.newly_uncontended_addresses.remove(&l.status.page_rc());
         }
     }
 

From 8b6c93099a7db3c42ef440494953e0735ebc1f9e Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 7 Aug 2022 21:53:02 +0900
Subject: [PATCH 0223/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index fe8f973c7ceb15..a6397453e5a961 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -565,7 +565,7 @@ impl ScheduleStage {
 
     #[inline(never)]
     fn unlock_after_execution(address_book: &mut AddressBook, lock_attempts: Vec<LockAttempt>) {
-        for mut l in lock_attempts {
+        for mut l in lock_attempts.into_iter() {
             let newly_uncontended_while_queued = address_book.unlock(&mut l);
             if newly_uncontended_while_queued {
                 address_book.newly_uncontended_addresses.insert(*l.status.page_rc());

From 383c0f6ac743dfd85f9b5221220bb680be004de5 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 7 Aug 2022 21:53:49 +0900
Subject: [PATCH 0224/3199] save

---
 scheduler/src/lib.rs | 1 +
 1 file changed, 1 insertion(+)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index a6397453e5a961..c415474e5f549b 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -568,6 +568,7 @@ impl ScheduleStage {
         for mut l in lock_attempts.into_iter() {
             let newly_uncontended_while_queued = address_book.unlock(&mut l);
             if newly_uncontended_while_queued {
+                let LockAttempt { status, .. } = *l;
                 address_book.newly_uncontended_addresses.insert(*l.status.page_rc());
             }
 

From 292c3eb26c05092b754c16556ff175f56554793b Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 7 Aug 2022 21:53:57 +0900
Subject: [PATCH 0225/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index c415474e5f549b..ad45d454d6902c 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -568,7 +568,7 @@ impl ScheduleStage {
         for mut l in lock_attempts.into_iter() {
             let newly_uncontended_while_queued = address_book.unlock(&mut l);
             if newly_uncontended_while_queued {
-                let LockAttempt { status, .. } = *l;
+                let LockAttempt { status, .. } = l;
                 address_book.newly_uncontended_addresses.insert(*l.status.page_rc());
             }
 

From 99f57cb8b24be5174955e8cc32d36f373af6be16 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 7 Aug 2022 21:54:44 +0900
Subject: [PATCH 0226/3199] save

---
 scheduler/src/lib.rs | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index ad45d454d6902c..fd177b55dd3e84 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -62,6 +62,13 @@ impl LockAttemptStatus {
         }
     }
 
+    fn take_page_rc(&mut self) -> MyRc<Page> {
+        match self {
+            LockAttemptStatus::BeforeLookup(_) => unreachable!(),
+            LockAttemptStatus::AfterLookup(_address, page) => page,
+        }
+    }
+
     fn page(&mut self) -> &mut Page {
         match self {
             LockAttemptStatus::BeforeLookup(_) => unreachable!(),
@@ -568,8 +575,7 @@ impl ScheduleStage {
         for mut l in lock_attempts.into_iter() {
             let newly_uncontended_while_queued = address_book.unlock(&mut l);
             if newly_uncontended_while_queued {
-                let LockAttempt { status, .. } = l;
-                address_book.newly_uncontended_addresses.insert(*l.status.page_rc());
+                address_book.newly_uncontended_addresses.insert(*l.status.take_page_rc());
             }
 
             // todo: mem::forget and panic in LockAttempt::drop()

From 812abd3af4d724d7ebb418da613f9b043015f026 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 7 Aug 2022 21:55:00 +0900
Subject: [PATCH 0227/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index fd177b55dd3e84..75625cc0e3110f 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -62,7 +62,7 @@ impl LockAttemptStatus {
         }
     }
 
-    fn take_page_rc(&mut self) -> MyRc<Page> {
+    fn take_page_rc(mut self) -> MyRc<Page> {
         match self {
             LockAttemptStatus::BeforeLookup(_) => unreachable!(),
             LockAttemptStatus::AfterLookup(_address, page) => page,

From 9841fe2c3b3a045736e4c351d7ebe876696eba16 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 7 Aug 2022 21:55:23 +0900
Subject: [PATCH 0228/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 75625cc0e3110f..19461ff4c0a17a 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -575,7 +575,7 @@ impl ScheduleStage {
         for mut l in lock_attempts.into_iter() {
             let newly_uncontended_while_queued = address_book.unlock(&mut l);
             if newly_uncontended_while_queued {
-                address_book.newly_uncontended_addresses.insert(*l.status.take_page_rc());
+                address_book.newly_uncontended_addresses.insert(l.status.take_page_rc());
             }
 
             // todo: mem::forget and panic in LockAttempt::drop()

From 35c8903a98feb2552d0cbb9e102f57b1fe9e1de7 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 7 Aug 2022 21:56:23 +0900
Subject: [PATCH 0229/3199] save

---
 scheduler/src/lib.rs | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 19461ff4c0a17a..8b1307d22a7c77 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -44,28 +44,28 @@ impl ExecutionEnvironment {
 #[derive(Clone, Debug)]
 enum LockAttemptStatus {
     BeforeLookup(Pubkey),
-    AfterLookup(Pubkey, MyRc<Page>),
+    AfterLookup(MyRc<Page>),
 }
 
 impl LockAttemptStatus {
     fn address2(&self) -> &Pubkey {
         match self {
             LockAttemptStatus::BeforeLookup(pubkey) => &pubkey,
-            LockAttemptStatus::AfterLookup(pubkey, _) => &pubkey,
+            LockAttemptStatus::AfterLookup(_) => unreachable!(),
         }
     }
 
     fn page_rc(&mut self) -> &MyRc<Page> {
         match self {
             LockAttemptStatus::BeforeLookup(_) => unreachable!(),
-            LockAttemptStatus::AfterLookup(_address, page) => page,
+            LockAttemptStatus::AfterLookup(page) => page,
         }
     }
 
     fn take_page_rc(mut self) -> MyRc<Page> {
         match self {
             LockAttemptStatus::BeforeLookup(_) => unreachable!(),
-            LockAttemptStatus::AfterLookup(_address, page) => page,
+            LockAttemptStatus::AfterLookup(page) => page,
         }
     }
 

From 3490ab3db5b18f38c1536c3084a34e5a6cf8bf09 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 7 Aug 2022 21:56:42 +0900
Subject: [PATCH 0230/3199] save

---
 scheduler/src/lib.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 8b1307d22a7c77..36b2866febe8e9 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -174,13 +174,13 @@ impl AddressBook {
         match self.book.entry(*address) {
             AddressMapEntry::Vacant(book_entry) => {
                 let page = ByAddress(MyRcInner::new(Page::new(CurrentUsage::renew(*requested_usage))));
-                *status = LockAttemptStatus::AfterLookup(*address, MyRc::clone(&page));
+                *status = LockAttemptStatus::AfterLookup(MyRc::clone(&page));
                 *is_success = true;
                 book_entry.insert(page);
             }
             AddressMapEntry::Occupied(mut book_entry) => {
                 let page = book_entry.get_mut();
-                *status = LockAttemptStatus::AfterLookup(*address, MyRc::clone(&page));
+                *status = LockAttemptStatus::AfterLookup(MyRc::clone(&page));
                 let mut page = unsafe { MyRcInner::get_mut_unchecked(page) };
 
                 match page.current_usage {

From 1cdbcb6887d2a743ca7468f35ba45d5d72a4d5fb Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 7 Aug 2022 21:56:52 +0900
Subject: [PATCH 0231/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 36b2866febe8e9..6935adb1ab5577 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -72,7 +72,7 @@ impl LockAttemptStatus {
     fn page(&mut self) -> &mut Page {
         match self {
             LockAttemptStatus::BeforeLookup(_) => unreachable!(),
-            LockAttemptStatus::AfterLookup(_address, page) => unsafe { MyRcInner::get_mut_unchecked(page) },
+            LockAttemptStatus::AfterLookup(page) => unsafe { MyRcInner::get_mut_unchecked(page) },
         }
     }
 }

From 6f9090a387d8e9a7cc88873c93bc4df2d204ccb3 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 7 Aug 2022 22:01:50 +0900
Subject: [PATCH 0232/3199] save

---
 scheduler/src/lib.rs | 52 ++++++++++++++++++++++++++++++++++++--------
 1 file changed, 43 insertions(+), 9 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 6935adb1ab5577..39bb1545d609e0 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -171,16 +171,50 @@ impl AddressBook {
         let LockAttempt {status, requested_usage, is_success} = attempt;
 
         let address = status.address2();
-        match self.book.entry(*address) {
-            AddressMapEntry::Vacant(book_entry) => {
-                let page = ByAddress(MyRcInner::new(Page::new(CurrentUsage::renew(*requested_usage))));
-                *status = LockAttemptStatus::AfterLookup(MyRc::clone(&page));
-                *is_success = true;
-                book_entry.insert(page);
+        match status {
+            LockAttemptStatus::BeforeLookup(ref address) {
+                match self.book.entry(address) {
+                    AddressMapEntry::Vacant(book_entry) => {
+                        let page = ByAddress(MyRcInner::new(Page::new(CurrentUsage::renew(*requested_usage))));
+                        *status = LockAttemptStatus::AfterLookup(MyRc::clone(&page));
+                        *is_success = true;
+                        book_entry.insert(page);
+                    }
+                    AddressMapEntry::Occupied(mut book_entry) => {
+                        let page = book_entry.get_mut();
+                        *status = LockAttemptStatus::AfterLookup(MyRc::clone(&page));
+                        let mut page = unsafe { MyRcInner::get_mut_unchecked(page) };
+
+                        match page.current_usage {
+                            CurrentUsage::Unused => {
+                                page.current_usage = CurrentUsage::renew(*requested_usage);
+                                *is_success = true;
+                            }
+                            CurrentUsage::Readonly(ref mut count) => match requested_usage {
+                                RequestedUsage::Readonly => {
+                                    *count += 1;
+                                    *is_success = true;
+                                }
+                                RequestedUsage::Writable => {
+                                    if from_runnable {
+                                        Self::remember_address_contention(&mut page, unique_weight);
+                                    }
+                                    *is_success = false;
+                                }
+                            },
+                            CurrentUsage::Writable => match requested_usage {
+                                RequestedUsage::Readonly | RequestedUsage::Writable => {
+                                    if from_runnable {
+                                        Self::remember_address_contention(&mut page, unique_weight);
+                                    }
+                                    *is_success = false;
+                                }
+                            },
+                        }
+                    }
+                }
             }
-            AddressMapEntry::Occupied(mut book_entry) => {
-                let page = book_entry.get_mut();
-                *status = LockAttemptStatus::AfterLookup(MyRc::clone(&page));
+            LockAttemptStatus::AfterLookup(page) {
                 let mut page = unsafe { MyRcInner::get_mut_unchecked(page) };
 
                 match page.current_usage {

From 81e2b791425aa1b51a9d2b8d30c04fcd2d7670be Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 7 Aug 2022 22:02:11 +0900
Subject: [PATCH 0233/3199] save

---
 scheduler/src/lib.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 39bb1545d609e0..a0cf94f00a7837 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -172,7 +172,7 @@ impl AddressBook {
 
         let address = status.address2();
         match status {
-            LockAttemptStatus::BeforeLookup(ref address) {
+            LockAttemptStatus::BeforeLookup(ref address) => {
                 match self.book.entry(address) {
                     AddressMapEntry::Vacant(book_entry) => {
                         let page = ByAddress(MyRcInner::new(Page::new(CurrentUsage::renew(*requested_usage))));
@@ -214,7 +214,7 @@ impl AddressBook {
                     }
                 }
             }
-            LockAttemptStatus::AfterLookup(page) {
+            LockAttemptStatus::AfterLookup(page) => {
                 let mut page = unsafe { MyRcInner::get_mut_unchecked(page) };
 
                 match page.current_usage {

From 4cfb00a3f0469a08c6d991f2f6d31839ef196b6e Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 7 Aug 2022 22:02:23 +0900
Subject: [PATCH 0234/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index a0cf94f00a7837..c085a54669eeee 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -173,7 +173,7 @@ impl AddressBook {
         let address = status.address2();
         match status {
             LockAttemptStatus::BeforeLookup(ref address) => {
-                match self.book.entry(address) {
+                match self.book.entry(&address) {
                     AddressMapEntry::Vacant(book_entry) => {
                         let page = ByAddress(MyRcInner::new(Page::new(CurrentUsage::renew(*requested_usage))));
                         *status = LockAttemptStatus::AfterLookup(MyRc::clone(&page));

From c888afb51d80fa82de8f4838cd1ad0394c64b5ac Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 7 Aug 2022 22:02:31 +0900
Subject: [PATCH 0235/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index c085a54669eeee..ed5af7af86b14d 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -172,7 +172,7 @@ impl AddressBook {
 
         let address = status.address2();
         match status {
-            LockAttemptStatus::BeforeLookup(ref address) => {
+            LockAttemptStatus::BeforeLookup(&address) => {
                 match self.book.entry(&address) {
                     AddressMapEntry::Vacant(book_entry) => {
                         let page = ByAddress(MyRcInner::new(Page::new(CurrentUsage::renew(*requested_usage))));

From 2def27780baeda642682eca12a98457d6448fcfe Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 7 Aug 2022 22:02:38 +0900
Subject: [PATCH 0236/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index ed5af7af86b14d..3513ebeacb8ff3 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -173,7 +173,7 @@ impl AddressBook {
         let address = status.address2();
         match status {
             LockAttemptStatus::BeforeLookup(&address) => {
-                match self.book.entry(&address) {
+                match self.book.entry(address) {
                     AddressMapEntry::Vacant(book_entry) => {
                         let page = ByAddress(MyRcInner::new(Page::new(CurrentUsage::renew(*requested_usage))));
                         *status = LockAttemptStatus::AfterLookup(MyRc::clone(&page));

From 7c18d448dec88b5bc898ca8a7e1a7c51df993e29 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 7 Aug 2022 22:02:47 +0900
Subject: [PATCH 0237/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 3513ebeacb8ff3..0bd05ddbe88a30 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -172,7 +172,7 @@ impl AddressBook {
 
         let address = status.address2();
         match status {
-            LockAttemptStatus::BeforeLookup(&address) => {
+            LockAttemptStatus::BeforeLookup(address) => {
                 match self.book.entry(address) {
                     AddressMapEntry::Vacant(book_entry) => {
                         let page = ByAddress(MyRcInner::new(Page::new(CurrentUsage::renew(*requested_usage))));

From 9a0d2f6ed16ca71eaa9b9b73e1e5fa96950119cc Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 7 Aug 2022 22:03:00 +0900
Subject: [PATCH 0238/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 0bd05ddbe88a30..73f1481f53c400 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -173,7 +173,7 @@ impl AddressBook {
         let address = status.address2();
         match status {
             LockAttemptStatus::BeforeLookup(address) => {
-                match self.book.entry(address) {
+                match self.book.entry(*address) {
                     AddressMapEntry::Vacant(book_entry) => {
                         let page = ByAddress(MyRcInner::new(Page::new(CurrentUsage::renew(*requested_usage))));
                         *status = LockAttemptStatus::AfterLookup(MyRc::clone(&page));

From c78b825f33faf1a62f66d19d96ae9c5ef51fc528 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 7 Aug 2022 22:04:06 +0900
Subject: [PATCH 0239/3199] save

---
 scheduler/src/lib.rs | 8 --------
 1 file changed, 8 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 73f1481f53c400..22cdbe9f6ed9da 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -48,13 +48,6 @@ enum LockAttemptStatus {
 }
 
 impl LockAttemptStatus {
-    fn address2(&self) -> &Pubkey {
-        match self {
-            LockAttemptStatus::BeforeLookup(pubkey) => &pubkey,
-            LockAttemptStatus::AfterLookup(_) => unreachable!(),
-        }
-    }
-
     fn page_rc(&mut self) -> &MyRc<Page> {
         match self {
             LockAttemptStatus::BeforeLookup(_) => unreachable!(),
@@ -170,7 +163,6 @@ impl AddressBook {
     ) {
         let LockAttempt {status, requested_usage, is_success} = attempt;
 
-        let address = status.address2();
         match status {
             LockAttemptStatus::BeforeLookup(address) => {
                 match self.book.entry(*address) {

From 527911e51dca01dea8604bca6e748e9ab55d0245 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 7 Aug 2022 22:19:53 +0900
Subject: [PATCH 0240/3199] save

---
 scheduler/src/lib.rs | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 22cdbe9f6ed9da..60ef7b18a6eed3 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -20,6 +20,8 @@ use {
 type MyRcInner<T> = std::sync::Arc<T>;
 type MyRc<T> = ByAddress<MyRcInner<T>>;
 
+unsafe impl<T> Send for MyRc<T>;
+
 #[derive(Debug)]
 pub struct ExecutionEnvironment {
     lock_attempts: Vec<LockAttempt>,

From d70bbc7f39d1c633dd143f8f207d0fe3fda7f635 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 7 Aug 2022 22:20:02 +0900
Subject: [PATCH 0241/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 60ef7b18a6eed3..1cab48dcce2d42 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -20,7 +20,7 @@ use {
 type MyRcInner<T> = std::sync::Arc<T>;
 type MyRc<T> = ByAddress<MyRcInner<T>>;
 
-unsafe impl<T> Send for MyRc<T>;
+unsafe impl<T> Send for MyRc<T> {}
 
 #[derive(Debug)]
 pub struct ExecutionEnvironment {

From 14686c38b1a9434f4756832dd359fb3d0aa77399 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 7 Aug 2022 22:20:45 +0900
Subject: [PATCH 0242/3199] save

---
 scheduler/src/lib.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 1cab48dcce2d42..45b115b4862941 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -18,9 +18,9 @@ use {
 };
 
 type MyRcInner<T> = std::sync::Arc<T>;
-type MyRc<T> = ByAddress<MyRcInner<T>>;
+struct MyRc<T>(ByAddress<MyRcInner<T>>);
 
-unsafe impl<T> Send for MyRc<T> {}
+//unsafe impl<T> Send for MyRc<T> {}
 
 #[derive(Debug)]
 pub struct ExecutionEnvironment {

From 0db5879b3153d8037130d71dc2b1bdc14f9d727e Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 7 Aug 2022 22:21:09 +0900
Subject: [PATCH 0243/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 45b115b4862941..320c9c4bfe38a9 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -67,7 +67,7 @@ impl LockAttemptStatus {
     fn page(&mut self) -> &mut Page {
         match self {
             LockAttemptStatus::BeforeLookup(_) => unreachable!(),
-            LockAttemptStatus::AfterLookup(page) => unsafe { MyRcInner::get_mut_unchecked(page) },
+            LockAttemptStatus::AfterLookup(page) => unsafe { MyRcInner::get_mut_unchecked(page.0) },
         }
     }
 }

From d7efd60932fd14629591b0c72efad291020480cb Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 7 Aug 2022 22:22:31 +0900
Subject: [PATCH 0244/3199] save

---
 scheduler/src/lib.rs | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 320c9c4bfe38a9..56319cad3d17c4 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -18,6 +18,7 @@ use {
 };
 
 type MyRcInner<T> = std::sync::Arc<T>;
+#[derive(Clone)]
 struct MyRc<T>(ByAddress<MyRcInner<T>>);
 
 //unsafe impl<T> Send for MyRc<T> {}
@@ -67,7 +68,7 @@ impl LockAttemptStatus {
     fn page(&mut self) -> &mut Page {
         match self {
             LockAttemptStatus::BeforeLookup(_) => unreachable!(),
-            LockAttemptStatus::AfterLookup(page) => unsafe { MyRcInner::get_mut_unchecked(page.0) },
+            LockAttemptStatus::AfterLookup(page) => unsafe { MyRcInner::get_mut_unchecked(&mut page.0) },
         }
     }
 }

From 31a687eca7db0bf9244f9a164fb77d4f30da9d04 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 7 Aug 2022 22:23:08 +0900
Subject: [PATCH 0245/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 56319cad3d17c4..ce7830b45a8d33 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -18,7 +18,7 @@ use {
 };
 
 type MyRcInner<T> = std::sync::Arc<T>;
-#[derive(Clone)]
+#[derive(Debug, Clone)]
 struct MyRc<T>(ByAddress<MyRcInner<T>>);
 
 //unsafe impl<T> Send for MyRc<T> {}

From d18fc35027ef23b53c9802a22d638e0dc44cf8f6 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 7 Aug 2022 22:23:36 +0900
Subject: [PATCH 0246/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index ce7830b45a8d33..a358c15c8363c9 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -18,7 +18,7 @@ use {
 };
 
 type MyRcInner<T> = std::sync::Arc<T>;
-#[derive(Debug, Clone)]
+#[derive(Debug)]
 struct MyRc<T>(ByAddress<MyRcInner<T>>);
 
 //unsafe impl<T> Send for MyRc<T> {}

From 06e8d0cbb96e027691722ac6220abe99d73276b6 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 7 Aug 2022 22:24:01 +0900
Subject: [PATCH 0247/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index a358c15c8363c9..52aa5a9668f4c9 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -18,7 +18,7 @@ use {
 };
 
 type MyRcInner<T> = std::sync::Arc<T>;
-#[derive(Debug)]
+#[derive(Debug, Eq, Hash, PartialEq)]
 struct MyRc<T>(ByAddress<MyRcInner<T>>);
 
 //unsafe impl<T> Send for MyRc<T> {}

From d8dc67311749a34dc2e4641355233e3e703d03a6 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 7 Aug 2022 22:24:25 +0900
Subject: [PATCH 0248/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 52aa5a9668f4c9..de0d3fe2af1ba4 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -18,7 +18,7 @@ use {
 };
 
 type MyRcInner<T> = std::sync::Arc<T>;
-#[derive(Debug, Eq, Hash, PartialEq)]
+#[derive(Debug, Eq, Hash, PartialEq, Clone)]
 struct MyRc<T>(ByAddress<MyRcInner<T>>);
 
 //unsafe impl<T> Send for MyRc<T> {}

From eef9abd2d1e6bc3b5b3abd2bae2b1befa99b0b6a Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 7 Aug 2022 22:25:45 +0900
Subject: [PATCH 0249/3199] save

---
 scheduler/src/lib.rs | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index de0d3fe2af1ba4..c9811c8863fd1e 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -18,9 +18,15 @@ use {
 };
 
 type MyRcInner<T> = std::sync::Arc<T>;
-#[derive(Debug, Eq, Hash, PartialEq, Clone)]
+#[derive(Debug, Eq, Hash, PartialEq)]
 struct MyRc<T>(ByAddress<MyRcInner<T>>);
 
+impl<T> Clone for MyRc<T> {
+    fn clone(&self) -> Self {
+        MyRc(ByAddress(MyRcInner::clone(*self.0)))
+    }
+}
+
 //unsafe impl<T> Send for MyRc<T> {}
 
 #[derive(Debug)]

From fe4e9470c87c68dc3578061c2341dd6bd7ea6c74 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 7 Aug 2022 22:26:04 +0900
Subject: [PATCH 0250/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index c9811c8863fd1e..30a032f3a38635 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -23,7 +23,7 @@ struct MyRc<T>(ByAddress<MyRcInner<T>>);
 
 impl<T> Clone for MyRc<T> {
     fn clone(&self) -> Self {
-        MyRc(ByAddress(MyRcInner::clone(*self.0)))
+        MyRc(ByAddress(MyRcInner::clone(self.0)))
     }
 }
 

From 4754fbd77c82e256f6e5433f3d39044d680c6b16 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 7 Aug 2022 22:26:15 +0900
Subject: [PATCH 0251/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 30a032f3a38635..42db52c0fedcf4 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -23,7 +23,7 @@ struct MyRc<T>(ByAddress<MyRcInner<T>>);
 
 impl<T> Clone for MyRc<T> {
     fn clone(&self) -> Self {
-        MyRc(ByAddress(MyRcInner::clone(self.0)))
+        MyRc(ByAddress(MyRcInner::clone(&self.0)))
     }
 }
 

From 58da0564dfcea265efaeb563a4e6f43ec768b646 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 7 Aug 2022 22:26:58 +0900
Subject: [PATCH 0252/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 42db52c0fedcf4..8d6326cde2f00c 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -176,7 +176,7 @@ impl AddressBook {
             LockAttemptStatus::BeforeLookup(address) => {
                 match self.book.entry(*address) {
                     AddressMapEntry::Vacant(book_entry) => {
-                        let page = ByAddress(MyRcInner::new(Page::new(CurrentUsage::renew(*requested_usage))));
+                        let page = MyRc(ByAddress(MyRcInner::new(Page::new(CurrentUsage::renew(*requested_usage)))));
                         *status = LockAttemptStatus::AfterLookup(MyRc::clone(&page));
                         *is_success = true;
                         book_entry.insert(page);

From 0260ee4a9adc01e719cebcb359b158802e992d69 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 7 Aug 2022 22:28:19 +0900
Subject: [PATCH 0253/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 8d6326cde2f00c..593ca036050f8c 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -184,7 +184,7 @@ impl AddressBook {
                     AddressMapEntry::Occupied(mut book_entry) => {
                         let page = book_entry.get_mut();
                         *status = LockAttemptStatus::AfterLookup(MyRc::clone(&page));
-                        let mut page = unsafe { MyRcInner::get_mut_unchecked(page) };
+                        let mut page = unsafe { MyRcInner::get_mut_unchecked(page.0) };
 
                         match page.current_usage {
                             CurrentUsage::Unused => {

From bcf3fb18c13f63f3da83c0f4b62555eebd62a34f Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 7 Aug 2022 22:28:32 +0900
Subject: [PATCH 0254/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 593ca036050f8c..b6125f69fd98bb 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -184,7 +184,7 @@ impl AddressBook {
                     AddressMapEntry::Occupied(mut book_entry) => {
                         let page = book_entry.get_mut();
                         *status = LockAttemptStatus::AfterLookup(MyRc::clone(&page));
-                        let mut page = unsafe { MyRcInner::get_mut_unchecked(page.0) };
+                        let mut page = unsafe { MyRcInner::get_mut_unchecked(&mut page.0) };
 
                         match page.current_usage {
                             CurrentUsage::Unused => {

From c3b327c357330c1f0bb37da090dae5621d1b75d0 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 7 Aug 2022 22:28:49 +0900
Subject: [PATCH 0255/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index b6125f69fd98bb..f74831e62539a7 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -216,7 +216,7 @@ impl AddressBook {
                 }
             }
             LockAttemptStatus::AfterLookup(page) => {
-                let mut page = unsafe { MyRcInner::get_mut_unchecked(page) };
+                let mut page = unsafe { MyRcInner::get_mut_unchecked(&mut page.0) };
 
                 match page.current_usage {
                     CurrentUsage::Unused => {

From 61ef2e5035a4d48b6d83b2e4410ed8fe671c5ecb Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 7 Aug 2022 22:29:04 +0900
Subject: [PATCH 0256/3199] save

---
 scheduler/src/lib.rs | 1 +
 1 file changed, 1 insertion(+)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index f74831e62539a7..177895be4ba278 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -454,6 +454,7 @@ impl ScheduleStage {
             .book
             .get(address)
             .unwrap()
+            .0
             .contended_unique_weights
     }
 

From 0e36c610dfe48a4f4a7872f0688714a9f282b281 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 7 Aug 2022 22:29:19 +0900
Subject: [PATCH 0257/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 177895be4ba278..91173384ac3273 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -463,7 +463,7 @@ impl ScheduleStage {
         let mut heaviest_weight: Option<UniqueWeight> = None;
         //trace!("n u a len(): {}", address_book.newly_uncontended_addresses.len());
         for page in address_book.newly_uncontended_addresses.iter() {
-            let newly_uncontended_unique_weights = &page.contended_unique_weights;
+            let newly_uncontended_unique_weights = &page.0.contended_unique_weights;
             if let Some(&weight) = newly_uncontended_unique_weights.last() {
                 if let Some(current_heaviest_weight) = heaviest_weight {
                     if weight > current_heaviest_weight {

From 2ddd4ed71f96c491dbcd4cda17a2c1badf4e88c3 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 7 Aug 2022 22:30:17 +0900
Subject: [PATCH 0258/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 91173384ac3273..b087ad4728495f 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -18,7 +18,7 @@ use {
 };
 
 type MyRcInner<T> = std::sync::Arc<T>;
-#[derive(Debug, Eq, Hash, PartialEq)]
+#[derive(Eq, Hash, PartialEq)]
 struct MyRc<T>(ByAddress<MyRcInner<T>>);
 
 impl<T> Clone for MyRc<T> {

From 426789004da4095e5c0cca2e396fd74fdc2d07cd Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 7 Aug 2022 22:30:53 +0900
Subject: [PATCH 0259/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index b087ad4728495f..9034b8a9d56d3c 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -19,7 +19,7 @@ use {
 
 type MyRcInner<T> = std::sync::Arc<T>;
 #[derive(Eq, Hash, PartialEq)]
-struct MyRc<T>(ByAddress<MyRcInner<T>>);
+struct MyRc<T: Eq>(ByAddress<MyRcInner<T>>);
 
 impl<T> Clone for MyRc<T> {
     fn clone(&self) -> Self {

From dd26e11e3b01cabbdc8274eed27fa03f872edee0 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 7 Aug 2022 22:31:07 +0900
Subject: [PATCH 0260/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 9034b8a9d56d3c..ff4b0fdb7434e1 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -19,7 +19,7 @@ use {
 
 type MyRcInner<T> = std::sync::Arc<T>;
 #[derive(Eq, Hash, PartialEq)]
-struct MyRc<T: Eq>(ByAddress<MyRcInner<T>>);
+struct MyRc<T>(pub ByAddress<MyRcInner<T>>);
 
 impl<T> Clone for MyRc<T> {
     fn clone(&self) -> Self {

From aa8890b7c0cbf851e0d16743f12948e71548a918 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 7 Aug 2022 22:31:41 +0900
Subject: [PATCH 0261/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index ff4b0fdb7434e1..89e571c795f6e1 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -18,7 +18,7 @@ use {
 };
 
 type MyRcInner<T> = std::sync::Arc<T>;
-#[derive(Eq, Hash, PartialEq)]
+#[derive(Hash, PartialEq)]
 struct MyRc<T>(pub ByAddress<MyRcInner<T>>);
 
 impl<T> Clone for MyRc<T> {

From e91b75dac240a15a63b4b0c6fa36b4e867baf6d1 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 7 Aug 2022 22:32:07 +0900
Subject: [PATCH 0262/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 89e571c795f6e1..90be3e3ac34707 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -18,7 +18,7 @@ use {
 };
 
 type MyRcInner<T> = std::sync::Arc<T>;
-#[derive(Hash, PartialEq)]
+#[derive(PartialEq)]
 struct MyRc<T>(pub ByAddress<MyRcInner<T>>);
 
 impl<T> Clone for MyRc<T> {

From 9546de2c70c9a96f12dc5c10199438eacfe7810e Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 7 Aug 2022 22:32:22 +0900
Subject: [PATCH 0263/3199] save

---
 scheduler/src/lib.rs | 1 +
 1 file changed, 1 insertion(+)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 90be3e3ac34707..264d0720024553 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -19,6 +19,7 @@ use {
 
 type MyRcInner<T> = std::sync::Arc<T>;
 #[derive(PartialEq)]
+#[derive(Debug)]
 struct MyRc<T>(pub ByAddress<MyRcInner<T>>);
 
 impl<T> Clone for MyRc<T> {

From 8b4731d141a588026d1a7efa27330c9608f96db2 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 7 Aug 2022 22:32:36 +0900
Subject: [PATCH 0264/3199] save

---
 scheduler/src/lib.rs | 1 +
 1 file changed, 1 insertion(+)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 264d0720024553..c23587f6f510c2 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -20,6 +20,7 @@ use {
 type MyRcInner<T> = std::sync::Arc<T>;
 #[derive(PartialEq)]
 #[derive(Debug)]
+#[derive(Eq)]
 struct MyRc<T>(pub ByAddress<MyRcInner<T>>);
 
 impl<T> Clone for MyRc<T> {

From d8fb60a948473d02ae6cc5e7c9e49d235f7260b1 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 7 Aug 2022 22:32:45 +0900
Subject: [PATCH 0265/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index c23587f6f510c2..413b71d57f68f5 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -18,9 +18,9 @@ use {
 };
 
 type MyRcInner<T> = std::sync::Arc<T>;
+#[derive(Eq)]
 #[derive(PartialEq)]
 #[derive(Debug)]
-#[derive(Eq)]
 struct MyRc<T>(pub ByAddress<MyRcInner<T>>);
 
 impl<T> Clone for MyRc<T> {

From 9744b4897a12f1872facc6d10abeb98be2abb3a6 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 7 Aug 2022 22:32:56 +0900
Subject: [PATCH 0266/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 413b71d57f68f5..e3d117ceb265ae 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -18,7 +18,7 @@ use {
 };
 
 type MyRcInner<T> = std::sync::Arc<T>;
-#[derive(Eq)]
+#[derive(Hash)]
 #[derive(PartialEq)]
 #[derive(Debug)]
 struct MyRc<T>(pub ByAddress<MyRcInner<T>>);

From ead8444e327684a49c16e63e92fffc0dfeef92bc Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 7 Aug 2022 22:33:07 +0900
Subject: [PATCH 0267/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index e3d117ceb265ae..878d62fe1330b9 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -18,7 +18,7 @@ use {
 };
 
 type MyRcInner<T> = std::sync::Arc<T>;
-#[derive(Hash)]
+#[derive(std::hash::Hash)]
 #[derive(PartialEq)]
 #[derive(Debug)]
 struct MyRc<T>(pub ByAddress<MyRcInner<T>>);

From 46be277755e1d6fed0edce03382f8b4ec9acb405 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 7 Aug 2022 22:33:29 +0900
Subject: [PATCH 0268/3199] save

---
 scheduler/src/lib.rs | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 878d62fe1330b9..e61d8ba5918d62 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -18,10 +18,7 @@ use {
 };
 
 type MyRcInner<T> = std::sync::Arc<T>;
-#[derive(std::hash::Hash)]
-#[derive(PartialEq)]
-#[derive(Debug)]
-struct MyRc<T>(pub ByAddress<MyRcInner<T>>);
+struct MyRc<T>{a: pub ByAddress<MyRcInner<T>>};
 
 impl<T> Clone for MyRc<T> {
     fn clone(&self) -> Self {

From ffe2a55b340d9c95341a4434013bd376debad82d Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 7 Aug 2022 22:33:51 +0900
Subject: [PATCH 0269/3199] save

---
 scheduler/src/lib.rs | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index e61d8ba5918d62..791b42c9547aeb 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -18,7 +18,8 @@ use {
 };
 
 type MyRcInner<T> = std::sync::Arc<T>;
-struct MyRc<T>{a: pub ByAddress<MyRcInner<T>>};
+#[derive(Debug, Hash)]
+struct MyRc<T>{pub a: ByAddress<MyRcInner<T>>};
 
 impl<T> Clone for MyRc<T> {
     fn clone(&self) -> Self {

From ad607af1470f72bff8aa5f65049624fc7c0f29b8 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 7 Aug 2022 22:34:47 +0900
Subject: [PATCH 0270/3199] save

---
 scheduler/src/lib.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 791b42c9547aeb..0cf567598c6a3a 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -18,8 +18,8 @@ use {
 };
 
 type MyRcInner<T> = std::sync::Arc<T>;
-#[derive(Debug, Hash)]
-struct MyRc<T>{pub a: ByAddress<MyRcInner<T>>};
+#[derive(Debug)]
+struct MyRc<T>(ByAddress<MyRcInner<T>>);
 
 impl<T> Clone for MyRc<T> {
     fn clone(&self) -> Self {

From e57e0eae239dbe2be999bb9816f1d444e1daa671 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 7 Aug 2022 22:35:27 +0900
Subject: [PATCH 0271/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 0cf567598c6a3a..2114830153b706 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -19,7 +19,7 @@ use {
 
 type MyRcInner<T> = std::sync::Arc<T>;
 #[derive(Debug)]
-struct MyRc<T>(ByAddress<MyRcInner<T>>);
+struct MyRc(ByAddress<MyRcInner<Page>>);
 
 impl<T> Clone for MyRc<T> {
     fn clone(&self) -> Self {

From 76ed46bb73c79620bacb682a2992546f8d0fb34b Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 7 Aug 2022 22:35:56 +0900
Subject: [PATCH 0272/3199] save

---
 scheduler/src/lib.rs | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 2114830153b706..d4fdfd0b41cb6f 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -21,7 +21,7 @@ type MyRcInner<T> = std::sync::Arc<T>;
 #[derive(Debug)]
 struct MyRc(ByAddress<MyRcInner<Page>>);
 
-impl<T> Clone for MyRc<T> {
+impl<T> Clone for MyRc {
     fn clone(&self) -> Self {
         MyRc(ByAddress(MyRcInner::clone(&self.0)))
     }
@@ -53,18 +53,18 @@ impl ExecutionEnvironment {
 #[derive(Clone, Debug)]
 enum LockAttemptStatus {
     BeforeLookup(Pubkey),
-    AfterLookup(MyRc<Page>),
+    AfterLookup(MyRc),
 }
 
 impl LockAttemptStatus {
-    fn page_rc(&mut self) -> &MyRc<Page> {
+    fn page_rc(&mut self) -> &MyRc {
         match self {
             LockAttemptStatus::BeforeLookup(_) => unreachable!(),
             LockAttemptStatus::AfterLookup(page) => page,
         }
     }
 
-    fn take_page_rc(mut self) -> MyRc<Page> {
+    fn take_page_rc(mut self) -> MyRc {
         match self {
             LockAttemptStatus::BeforeLookup(_) => unreachable!(),
             LockAttemptStatus::AfterLookup(page) => page,
@@ -150,9 +150,9 @@ impl Page {
     }
 }
 
-type AddressMap = std::collections::HashMap<Pubkey, MyRc<Page>>;
+type AddressMap = std::collections::HashMap<Pubkey, MyRc>;
 use by_address::ByAddress;
-type AddressSet = std::collections::HashSet<MyRc<Page>>;
+type AddressSet = std::collections::HashSet<MyRc>;
 type AddressMapEntry<'a, K, V> = std::collections::hash_map::Entry<'a, K, V>;
 
 // needs ttl mechanism and prune

From da9dea3e05756f521dcd16b40523c97f612b18ab Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 7 Aug 2022 22:36:08 +0900
Subject: [PATCH 0273/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index d4fdfd0b41cb6f..674baf264a6909 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -21,7 +21,7 @@ type MyRcInner<T> = std::sync::Arc<T>;
 #[derive(Debug)]
 struct MyRc(ByAddress<MyRcInner<Page>>);
 
-impl<T> Clone for MyRc {
+impl Clone for MyRc {
     fn clone(&self) -> Self {
         MyRc(ByAddress(MyRcInner::clone(&self.0)))
     }

From e489372cebba8d5036f5d0dc725f9f63f259cb2b Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 7 Aug 2022 22:36:30 +0900
Subject: [PATCH 0274/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 674baf264a6909..6502ff81d17158 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -18,7 +18,7 @@ use {
 };
 
 type MyRcInner<T> = std::sync::Arc<T>;
-#[derive(Debug)]
+#[derive(Debug, Hash)]
 struct MyRc(ByAddress<MyRcInner<Page>>);
 
 impl Clone for MyRc {

From 8cadb1978a23c2b497a47c5b6c2a0c2ba3166207 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 7 Aug 2022 22:36:42 +0900
Subject: [PATCH 0275/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 6502ff81d17158..de2a93d27124e0 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -18,7 +18,7 @@ use {
 };
 
 type MyRcInner<T> = std::sync::Arc<T>;
-#[derive(Debug, Hash)]
+#[derive(Debug, Hash, PartialEq)]
 struct MyRc(ByAddress<MyRcInner<Page>>);
 
 impl Clone for MyRc {

From cac9045217c39a8a8e991696e368e9751baf03ef Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 7 Aug 2022 22:36:48 +0900
Subject: [PATCH 0276/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index de2a93d27124e0..c3304baef7c21a 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -18,7 +18,7 @@ use {
 };
 
 type MyRcInner<T> = std::sync::Arc<T>;
-#[derive(Debug, Hash, PartialEq)]
+#[derive(Debug, Hash, PartialEq, Eq)]
 struct MyRc(ByAddress<MyRcInner<Page>>);
 
 impl Clone for MyRc {

From 59f5dcd24f0918adcce2093e3c2b2063144d4889 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 7 Aug 2022 22:37:02 +0900
Subject: [PATCH 0277/3199] save

---
 scheduler/src/lib.rs | 8 +-------
 1 file changed, 1 insertion(+), 7 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index c3304baef7c21a..26c33727e2b705 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -18,15 +18,9 @@ use {
 };
 
 type MyRcInner<T> = std::sync::Arc<T>;
-#[derive(Debug, Hash, PartialEq, Eq)]
+#[derive(Debug, Hash, PartialEq, Eq, Clone)]
 struct MyRc(ByAddress<MyRcInner<Page>>);
 
-impl Clone for MyRc {
-    fn clone(&self) -> Self {
-        MyRc(ByAddress(MyRcInner::clone(&self.0)))
-    }
-}
-
 //unsafe impl<T> Send for MyRc<T> {}
 
 #[derive(Debug)]

From 32e769c269671f99b9f774362b44e57d0569a873 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 7 Aug 2022 22:38:55 +0900
Subject: [PATCH 0278/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 26c33727e2b705..c710108ffdac80 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -17,7 +17,7 @@ use {
     },
 };
 
-type MyRcInner<T> = std::sync::Arc<T>;
+type MyRcInner<T> = std::rc::Rc<T>;
 #[derive(Debug, Hash, PartialEq, Eq, Clone)]
 struct MyRc(ByAddress<MyRcInner<Page>>);
 

From b0a9ee3c9081b26f1935c09180b1372b54b08265 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 7 Aug 2022 22:39:23 +0900
Subject: [PATCH 0279/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index c710108ffdac80..79be70d6726762 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -21,7 +21,7 @@ type MyRcInner<T> = std::rc::Rc<T>;
 #[derive(Debug, Hash, PartialEq, Eq, Clone)]
 struct MyRc(ByAddress<MyRcInner<Page>>);
 
-//unsafe impl<T> Send for MyRc<T> {}
+unsafe impl Send for MyRc {}
 
 #[derive(Debug)]
 pub struct ExecutionEnvironment {

From 86090de2757c0677d1cd64ba92a601c67c53d1f0 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 7 Aug 2022 22:41:01 +0900
Subject: [PATCH 0280/3199] save

---
 scheduler/src/lib.rs | 1 -
 1 file changed, 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 79be70d6726762..01a9a8323d28a1 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1,5 +1,4 @@
 #![feature(map_first_last)]
-#![feature(get_mut_unchecked)]
 
 use {
     atomic_enum::atomic_enum,

From 88782775763931f2c498eb58d99be4de8133de95 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 7 Aug 2022 22:41:15 +0900
Subject: [PATCH 0281/3199] save

---
 scheduler/src/lib.rs | 1 +
 1 file changed, 1 insertion(+)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 01a9a8323d28a1..79be70d6726762 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1,4 +1,5 @@
 #![feature(map_first_last)]
+#![feature(get_mut_unchecked)]
 
 use {
     atomic_enum::atomic_enum,

From c34b9f80f9c9ace06253a96fb8b07d82b20be818 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 7 Aug 2022 22:41:50 +0900
Subject: [PATCH 0282/3199] save

---
 transaction-scheduler-bench/src/main.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/transaction-scheduler-bench/src/main.rs b/transaction-scheduler-bench/src/main.rs
index 15ddb048f36675..572d2ab14fd8a7 100644
--- a/transaction-scheduler-bench/src/main.rs
+++ b/transaction-scheduler-bench/src/main.rs
@@ -339,7 +339,7 @@ fn handle_transaction_batch(
             .unwrap();
 }
 
-const NUM_SENDERS: usize = 1;
+const NUM_SENDERS: usize = 2;
 
 fn spawn_packet_senders(
     metrics: Arc<TransactionSchedulerBenchMetrics>,

From 2ee4642e73b8c21efa40746bb07091af51613a6d Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 7 Aug 2022 22:48:31 +0900
Subject: [PATCH 0283/3199] save

---
 transaction-scheduler-bench/src/main.rs | 14 +++++---------
 1 file changed, 5 insertions(+), 9 deletions(-)

diff --git a/transaction-scheduler-bench/src/main.rs b/transaction-scheduler-bench/src/main.rs
index 572d2ab14fd8a7..006b5cd8003ed5 100644
--- a/transaction-scheduler-bench/src/main.rs
+++ b/transaction-scheduler-bench/src/main.rs
@@ -339,7 +339,7 @@ fn handle_transaction_batch(
             .unwrap();
 }
 
-const NUM_SENDERS: usize = 2;
+const NUM_SENDERS: usize = 1;
 
 fn spawn_packet_senders(
     metrics: Arc<TransactionSchedulerBenchMetrics>,
@@ -478,17 +478,13 @@ fn build_packet(
             read_account_metas.chain(write_account_metas).collect(),
         ),
     ];
-    let versioned_transaction = VersionedTransaction::from(Transaction::new_with_payer(
+    let transaction = Transaction::new_with_payer(
         &ixs,
         Some(&sending_keypair.pubkey()),
-    ));
+    );
 
-    let sanitized_tx = SanitizedTransaction::try_create(
-        versioned_transaction,
-        solana_sdk::transaction::MessageHash::Compute,
-        None,
-        solana_sdk::transaction::SimpleAddressLoader::Disabled,
-        true, // require_static_program_ids
+    let sanitized_tx = SanitizedTransaction::try_from_legacy_transaction(
+        transaction,
     )
     .unwrap();
 

From 6465fc52fb4928daf13ecceb574470c0f8de4e68 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 7 Aug 2022 22:50:15 +0900
Subject: [PATCH 0284/3199] save

---
 sdk/src/transaction/sanitized.rs | 2 --
 1 file changed, 2 deletions(-)

diff --git a/sdk/src/transaction/sanitized.rs b/sdk/src/transaction/sanitized.rs
index 35a379f724da0a..46c095703c83bf 100644
--- a/sdk/src/transaction/sanitized.rs
+++ b/sdk/src/transaction/sanitized.rs
@@ -145,8 +145,6 @@ impl SanitizedTransaction {
     }
 
     pub fn try_from_legacy_transaction(tx: Transaction) -> Result<Self> {
-        tx.sanitize()?;
-
         Ok(Self {
             message_hash: tx.message.hash(),
             message: SanitizedMessage::Legacy(tx.message),

From ab21a511823787af2991d065b79971caa2455cd7 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 7 Aug 2022 22:58:52 +0900
Subject: [PATCH 0285/3199] save

---
 sdk/program/src/message/legacy.rs | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/sdk/program/src/message/legacy.rs b/sdk/program/src/message/legacy.rs
index 8b90322400253f..1faea0e64caaab 100644
--- a/sdk/program/src/message/legacy.rs
+++ b/sdk/program/src/message/legacy.rs
@@ -467,8 +467,9 @@ impl Message {
     /// Compute the blake3 hash of this transaction's message.
     #[cfg(not(target_os = "solana"))]
     pub fn hash(&self) -> Hash {
-        let message_bytes = self.serialize();
-        Self::hash_raw_message(&message_bytes)
+        //let message_bytes = self.serialize();
+        //Self::hash_raw_message(&message_bytes)
+        Hash::default()
     }
 
     /// Compute the blake3 hash of a raw transaction message.

From 5637c62f867df60818afc0d602acf5e4b3a4b5f8 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 7 Aug 2022 23:16:39 +0900
Subject: [PATCH 0286/3199] save

---
 transaction-scheduler-bench/src/main.rs | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/transaction-scheduler-bench/src/main.rs b/transaction-scheduler-bench/src/main.rs
index 006b5cd8003ed5..d0933d65354692 100644
--- a/transaction-scheduler-bench/src/main.rs
+++ b/transaction-scheduler-bench/src/main.rs
@@ -117,6 +117,7 @@ struct PacketSendingConfig {
 }
 
 fn spawn_unified_scheduler(
+        num_execution_threads: usize,
         packet_batch_receiver: Receiver<BatchSenderMessage>,
         transaction_batch_senders: Vec<Sender<TransactionBatchMessage>>,
         //completed_transaction_receiver: Receiver<CompletedTransactionMessage>,
@@ -130,7 +131,7 @@ fn spawn_unified_scheduler(
         let mut address_book = solana_scheduler::AddressBook::default();
 
         solana_scheduler::ScheduleStage::run(
-            100,
+            num_execution_threads * 10,
             &mut runnable_queue,
             &mut contended_queue,
             &mut address_book,
@@ -168,6 +169,7 @@ fn main() {
 
     // Spawns and runs the scheduler thread
     let scheduler_handle = spawn_unified_scheduler(
+        num_execution_threads,
         packet_batch_receiver,
         transaction_batch_senders,
         bank_forks,

From 903a9a45676a9179984205473b2f071097dd3f47 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 7 Aug 2022 23:18:04 +0900
Subject: [PATCH 0287/3199] save

---
 transaction-scheduler-bench/src/main.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/transaction-scheduler-bench/src/main.rs b/transaction-scheduler-bench/src/main.rs
index d0933d65354692..ef5b44b28aa650 100644
--- a/transaction-scheduler-bench/src/main.rs
+++ b/transaction-scheduler-bench/src/main.rs
@@ -131,7 +131,7 @@ fn spawn_unified_scheduler(
         let mut address_book = solana_scheduler::AddressBook::default();
 
         solana_scheduler::ScheduleStage::run(
-            num_execution_threads * 10,
+            num_execution_threads * 100,
             &mut runnable_queue,
             &mut contended_queue,
             &mut address_book,

From a0658e9efb9164ab8a03270e078830dfee9b4d5d Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 7 Aug 2022 23:38:51 +0900
Subject: [PATCH 0288/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 79be70d6726762..9fa0476fc52d57 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -128,7 +128,7 @@ pub enum RequestedUsage {
 #[derive(Debug)]
 struct Page {
     current_usage: CurrentUsage,
-    contended_unique_weights: std::collections::BTreeSet<UniqueWeight>,
+    contended_unique_weights: std::collections::BinaryHeap<UniqueWeight>,
     //next_scheduled_task // reserved_task guaranteed_task
     //loaded account from Accounts db
     //comulative_cu for qos; i.e. track serialized cumulative keyed by addresses and bail out block

From 0f21cdb484680244f8c81a6454988c32962d55dd Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 7 Aug 2022 23:40:49 +0900
Subject: [PATCH 0289/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 9fa0476fc52d57..01ad83eae23bfc 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -458,7 +458,7 @@ impl ScheduleStage {
         //trace!("n u a len(): {}", address_book.newly_uncontended_addresses.len());
         for page in address_book.newly_uncontended_addresses.iter() {
             let newly_uncontended_unique_weights = &page.0.contended_unique_weights;
-            if let Some(&weight) = newly_uncontended_unique_weights.last() {
+            if let Some(&weight) = newly_uncontended_unique_weights.max() {
                 if let Some(current_heaviest_weight) = heaviest_weight {
                     if weight > current_heaviest_weight {
                         heaviest_weight = Some(weight);

From 97e62073ec49995015f2fb53b863e451d5b3f1d5 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 7 Aug 2022 23:41:07 +0900
Subject: [PATCH 0290/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 01ad83eae23bfc..da13f2005b62b2 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -458,7 +458,7 @@ impl ScheduleStage {
         //trace!("n u a len(): {}", address_book.newly_uncontended_addresses.len());
         for page in address_book.newly_uncontended_addresses.iter() {
             let newly_uncontended_unique_weights = &page.0.contended_unique_weights;
-            if let Some(&weight) = newly_uncontended_unique_weights.max() {
+            if let Some(&weight) = newly_uncontended_unique_weights.pop() {
                 if let Some(current_heaviest_weight) = heaviest_weight {
                     if weight > current_heaviest_weight {
                         heaviest_weight = Some(weight);

From f160fe91817ff18018be9259ef1c21d8808744c8 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 7 Aug 2022 23:42:42 +0900
Subject: [PATCH 0291/3199] save

---
 scheduler/src/lib.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index da13f2005b62b2..79be70d6726762 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -128,7 +128,7 @@ pub enum RequestedUsage {
 #[derive(Debug)]
 struct Page {
     current_usage: CurrentUsage,
-    contended_unique_weights: std::collections::BinaryHeap<UniqueWeight>,
+    contended_unique_weights: std::collections::BTreeSet<UniqueWeight>,
     //next_scheduled_task // reserved_task guaranteed_task
     //loaded account from Accounts db
     //comulative_cu for qos; i.e. track serialized cumulative keyed by addresses and bail out block
@@ -458,7 +458,7 @@ impl ScheduleStage {
         //trace!("n u a len(): {}", address_book.newly_uncontended_addresses.len());
         for page in address_book.newly_uncontended_addresses.iter() {
             let newly_uncontended_unique_weights = &page.0.contended_unique_weights;
-            if let Some(&weight) = newly_uncontended_unique_weights.pop() {
+            if let Some(&weight) = newly_uncontended_unique_weights.last() {
                 if let Some(current_heaviest_weight) = heaviest_weight {
                     if weight > current_heaviest_weight {
                         heaviest_weight = Some(weight);

From 7f2d37cd0e0e3ed6395e27b874210375ae2f8f0a Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 7 Aug 2022 23:44:27 +0900
Subject: [PATCH 0292/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 79be70d6726762..919c1fb2ee75b4 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -455,7 +455,7 @@ impl ScheduleStage {
     #[inline(never)]
     fn get_weight_from_contended(address_book: &AddressBook) -> Option<UniqueWeight> {
         let mut heaviest_weight: Option<UniqueWeight> = None;
-        //trace!("n u a len(): {}", address_book.newly_uncontended_addresses.len());
+        trace!("n u a len(): {}", address_book.newly_uncontended_addresses.len());
         for page in address_book.newly_uncontended_addresses.iter() {
             let newly_uncontended_unique_weights = &page.0.contended_unique_weights;
             if let Some(&weight) = newly_uncontended_unique_weights.last() {

From 50b0aa9957fa5b4cf03901f64edb52e92b78fcc5 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 7 Aug 2022 23:44:34 +0900
Subject: [PATCH 0293/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 919c1fb2ee75b4..6a953eadaf59d2 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -455,7 +455,7 @@ impl ScheduleStage {
     #[inline(never)]
     fn get_weight_from_contended(address_book: &AddressBook) -> Option<UniqueWeight> {
         let mut heaviest_weight: Option<UniqueWeight> = None;
-        trace!("n u a len(): {}", address_book.newly_uncontended_addresses.len());
+        error!("n u a len(): {}", address_book.newly_uncontended_addresses.len());
         for page in address_book.newly_uncontended_addresses.iter() {
             let newly_uncontended_unique_weights = &page.0.contended_unique_weights;
             if let Some(&weight) = newly_uncontended_unique_weights.last() {

From 10c8eff342a2599b027fc051e925af7471233fa5 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 7 Aug 2022 23:47:18 +0900
Subject: [PATCH 0294/3199] save

---
 scheduler/src/lib.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 6a953eadaf59d2..7c2515c20a3a51 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -578,7 +578,7 @@ impl ScheduleStage {
             address_book.forget_address_contention(&unique_weight, &mut l);
 
             // revert because now contended again
-            address_book.newly_uncontended_addresses.remove(&l.status.page_rc());
+            address_book.newly_uncontended_addresses.remove(l.status.page_rc());
         }
     }
 
@@ -593,7 +593,7 @@ impl ScheduleStage {
 
             // revert because now contended again
             if !from_runnable {
-                address_book.newly_uncontended_addresses.remove(&l.status.page_rc());
+                address_book.newly_uncontended_addresses.remove(l.status.page_rc());
             }
 
             // todo: mem::forget and panic in LockAttempt::drop()

From 19fafb013f8b643d0ab53459d6cde9d4afdf3eed Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 7 Aug 2022 23:51:51 +0900
Subject: [PATCH 0295/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 7c2515c20a3a51..01a868987cfc8e 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -146,7 +146,7 @@ impl Page {
 
 type AddressMap = std::collections::HashMap<Pubkey, MyRc>;
 use by_address::ByAddress;
-type AddressSet = std::collections::HashSet<MyRc>;
+type AddressSet = std::collections::HashSet<ByAddress<MyRcInner<Page>>>;
 type AddressMapEntry<'a, K, V> = std::collections::hash_map::Entry<'a, K, V>;
 
 // needs ttl mechanism and prune

From aca1ce519aa10e77063658c9699eea03946c196b Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 7 Aug 2022 23:52:06 +0900
Subject: [PATCH 0296/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 01a868987cfc8e..b52f5aef99087a 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -578,7 +578,7 @@ impl ScheduleStage {
             address_book.forget_address_contention(&unique_weight, &mut l);
 
             // revert because now contended again
-            address_book.newly_uncontended_addresses.remove(l.status.page_rc());
+            address_book.newly_uncontended_addresses.remove(l.status.page_rc().0);
         }
     }
 

From d16b3eb713d441cb7552a09bf9a801368ee0c193 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 7 Aug 2022 23:52:14 +0900
Subject: [PATCH 0297/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index b52f5aef99087a..a16301d969fc2f 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -578,7 +578,7 @@ impl ScheduleStage {
             address_book.forget_address_contention(&unique_weight, &mut l);
 
             // revert because now contended again
-            address_book.newly_uncontended_addresses.remove(l.status.page_rc().0);
+            address_book.newly_uncontended_addresses.remove(&l.status.page_rc().0);
         }
     }
 

From e4ac87d779d58345968febe26646ea97be0368e9 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 7 Aug 2022 23:52:23 +0900
Subject: [PATCH 0298/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index a16301d969fc2f..9587b1b9c69f0c 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -593,7 +593,7 @@ impl ScheduleStage {
 
             // revert because now contended again
             if !from_runnable {
-                address_book.newly_uncontended_addresses.remove(l.status.page_rc());
+                address_book.newly_uncontended_addresses.remove(&l.status.page_rc().0);
             }
 
             // todo: mem::forget and panic in LockAttempt::drop()

From 5cbf63a162f0d396262c93b09d26c59370d710cb Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 7 Aug 2022 23:52:35 +0900
Subject: [PATCH 0299/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 9587b1b9c69f0c..19d743490ca4fb 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -605,7 +605,7 @@ impl ScheduleStage {
         for mut l in lock_attempts.into_iter() {
             let newly_uncontended_while_queued = address_book.unlock(&mut l);
             if newly_uncontended_while_queued {
-                address_book.newly_uncontended_addresses.insert(l.status.take_page_rc());
+                address_book.newly_uncontended_addresses.insert(l.status.take_page_rc().0);
             }
 
             // todo: mem::forget and panic in LockAttempt::drop()

From f711a22bb8601caea907d897a0f30bf29d007251 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 7 Aug 2022 23:54:02 +0900
Subject: [PATCH 0300/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 19d743490ca4fb..97235543784f73 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -17,7 +17,7 @@ use {
     },
 };
 
-type MyRcInner<T> = std::rc::Rc<T>;
+type MyRcInner<T> = std::sync::Arc<T>;
 #[derive(Debug, Hash, PartialEq, Eq, Clone)]
 struct MyRc(ByAddress<MyRcInner<Page>>);
 

From f707f48f7ca95f5e59ded690a2c1fd896e79f3e4 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 7 Aug 2022 23:56:52 +0900
Subject: [PATCH 0301/3199] save

---
 scheduler/src/lib.rs | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 97235543784f73..0916cc41dab6f0 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -593,7 +593,9 @@ impl ScheduleStage {
 
             // revert because now contended again
             if !from_runnable {
+                error!("n u a len() before: {}", address_book.newly_uncontended_addresses.len());
                 address_book.newly_uncontended_addresses.remove(&l.status.page_rc().0);
+                error!("n u a len() after: {}", address_book.newly_uncontended_addresses.len());
             }
 
             // todo: mem::forget and panic in LockAttempt::drop()

From 42f15a58c63c61014a4410c4ca775f94a7b124a6 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 8 Aug 2022 09:43:03 +0900
Subject: [PATCH 0302/3199] save

---
 scheduler/src/lib.rs | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 0916cc41dab6f0..15fe034c13d52a 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -177,7 +177,10 @@ impl AddressBook {
                     }
                     AddressMapEntry::Occupied(mut book_entry) => {
                         let page = book_entry.get_mut();
-                        *status = LockAttemptStatus::AfterLookup(MyRc::clone(&page));
+                        let cloned_page = MyRc::clone(&page);
+                        dbg!((page, cloned_page));
+                        assert_eq!(page, cloned_page);
+                        *status = LockAttemptStatus::AfterLookup(cloned_page);
                         let mut page = unsafe { MyRcInner::get_mut_unchecked(&mut page.0) };
 
                         match page.current_usage {

From 40f2619edf9227df4ee680d383dd61ae283f7e15 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 8 Aug 2022 09:43:16 +0900
Subject: [PATCH 0303/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 15fe034c13d52a..acaaf708465820 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -179,7 +179,7 @@ impl AddressBook {
                         let page = book_entry.get_mut();
                         let cloned_page = MyRc::clone(&page);
                         dbg!((page, cloned_page));
-                        assert_eq!(page, cloned_page);
+                        assert_eq!(*page, cloned_page);
                         *status = LockAttemptStatus::AfterLookup(cloned_page);
                         let mut page = unsafe { MyRcInner::get_mut_unchecked(&mut page.0) };
 

From 2731f1715b7584ca1e85a61faee21210095cb3cf Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 8 Aug 2022 09:43:26 +0900
Subject: [PATCH 0304/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index acaaf708465820..7aba63acd20b85 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -179,7 +179,7 @@ impl AddressBook {
                         let page = book_entry.get_mut();
                         let cloned_page = MyRc::clone(&page);
                         dbg!((page, cloned_page));
-                        assert_eq!(*page, cloned_page);
+                        assert_eq!(&*page, &cloned_page);
                         *status = LockAttemptStatus::AfterLookup(cloned_page);
                         let mut page = unsafe { MyRcInner::get_mut_unchecked(&mut page.0) };
 

From a04997bfc7d30bfb8207fef39ca222b71f391401 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 8 Aug 2022 09:43:32 +0900
Subject: [PATCH 0305/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 7aba63acd20b85..1e36cc9406b4ba 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -178,7 +178,7 @@ impl AddressBook {
                     AddressMapEntry::Occupied(mut book_entry) => {
                         let page = book_entry.get_mut();
                         let cloned_page = MyRc::clone(&page);
-                        dbg!((page, cloned_page));
+                        dbg!(&(page, cloned_page));
                         assert_eq!(&*page, &cloned_page);
                         *status = LockAttemptStatus::AfterLookup(cloned_page);
                         let mut page = unsafe { MyRcInner::get_mut_unchecked(&mut page.0) };

From ab522f1ea6c7214ea6e2e761127f9e6338614ee7 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 8 Aug 2022 09:43:40 +0900
Subject: [PATCH 0306/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 1e36cc9406b4ba..7056488274446c 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -179,7 +179,7 @@ impl AddressBook {
                         let page = book_entry.get_mut();
                         let cloned_page = MyRc::clone(&page);
                         dbg!(&(page, cloned_page));
-                        assert_eq!(&*page, &cloned_page);
+                        //assert_eq!(&*page, &cloned_page);
                         *status = LockAttemptStatus::AfterLookup(cloned_page);
                         let mut page = unsafe { MyRcInner::get_mut_unchecked(&mut page.0) };
 

From 1523a693e110f716907f919ee8e866af3a584516 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 8 Aug 2022 09:43:54 +0900
Subject: [PATCH 0307/3199] save

---
 scheduler/src/lib.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 7056488274446c..dc3fd39a3aed2d 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -178,8 +178,8 @@ impl AddressBook {
                     AddressMapEntry::Occupied(mut book_entry) => {
                         let page = book_entry.get_mut();
                         let cloned_page = MyRc::clone(&page);
-                        dbg!(&(page, cloned_page));
-                        //assert_eq!(&*page, &cloned_page);
+                        dbg!((&page, &cloned_page));
+                        assert_eq!(&*page, &cloned_page);
                         *status = LockAttemptStatus::AfterLookup(cloned_page);
                         let mut page = unsafe { MyRcInner::get_mut_unchecked(&mut page.0) };
 

From b441f8fae4d439c8cf9c79d9a53bf86adceae140 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 8 Aug 2022 09:47:09 +0900
Subject: [PATCH 0308/3199] save

---
 scheduler/src/lib.rs | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index dc3fd39a3aed2d..dd42817e2026f9 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -178,8 +178,12 @@ impl AddressBook {
                     AddressMapEntry::Occupied(mut book_entry) => {
                         let page = book_entry.get_mut();
                         let cloned_page = MyRc::clone(&page);
-                        dbg!((&page, &cloned_page));
-                        assert_eq!(&*page, &cloned_page);
+                        //dbg!((&page, &cloned_page));
+                        //assert_eq!(&*page, &cloned_page);
+                        let mut a = HashSet::new();
+                        a.insert(page);
+                        a.insert(cloned_page);
+                        panic!("{:?}", a);
                         *status = LockAttemptStatus::AfterLookup(cloned_page);
                         let mut page = unsafe { MyRcInner::get_mut_unchecked(&mut page.0) };
 

From e6dbdd35f598aff09fb189209590b3c183477959 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 8 Aug 2022 09:47:23 +0900
Subject: [PATCH 0309/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index dd42817e2026f9..2bea6cc40f6f2c 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -180,7 +180,7 @@ impl AddressBook {
                         let cloned_page = MyRc::clone(&page);
                         //dbg!((&page, &cloned_page));
                         //assert_eq!(&*page, &cloned_page);
-                        let mut a = HashSet::new();
+                        let mut a = std::collections::HashSet::new();
                         a.insert(page);
                         a.insert(cloned_page);
                         panic!("{:?}", a);

From 3991daf75291e213c21cb20ef0f8144b449949ad Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 8 Aug 2022 09:47:39 +0900
Subject: [PATCH 0310/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 2bea6cc40f6f2c..bf8c5b5d7848e3 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -181,7 +181,7 @@ impl AddressBook {
                         //dbg!((&page, &cloned_page));
                         //assert_eq!(&*page, &cloned_page);
                         let mut a = std::collections::HashSet::new();
-                        a.insert(page);
+                        a.insert(page.clone());
                         a.insert(cloned_page);
                         panic!("{:?}", a);
                         *status = LockAttemptStatus::AfterLookup(cloned_page);

From beec49f3ac174704cafac9cce584ce19611f03eb Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 8 Aug 2022 09:48:56 +0900
Subject: [PATCH 0311/3199] save

---
 scheduler/src/lib.rs                    | 8 ++++----
 transaction-scheduler-bench/src/main.rs | 2 +-
 2 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index bf8c5b5d7848e3..d3af33b759fa2c 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -180,10 +180,10 @@ impl AddressBook {
                         let cloned_page = MyRc::clone(&page);
                         //dbg!((&page, &cloned_page));
                         //assert_eq!(&*page, &cloned_page);
-                        let mut a = std::collections::HashSet::new();
-                        a.insert(page.clone());
-                        a.insert(cloned_page);
-                        panic!("{:?}", a);
+                        //let mut a = std::collections::HashSet::new();
+                        //a.insert(page.clone());
+                        //a.insert(cloned_page);
+                        //panic!("{:?}", a);
                         *status = LockAttemptStatus::AfterLookup(cloned_page);
                         let mut page = unsafe { MyRcInner::get_mut_unchecked(&mut page.0) };
 
diff --git a/transaction-scheduler-bench/src/main.rs b/transaction-scheduler-bench/src/main.rs
index ef5b44b28aa650..d0933d65354692 100644
--- a/transaction-scheduler-bench/src/main.rs
+++ b/transaction-scheduler-bench/src/main.rs
@@ -131,7 +131,7 @@ fn spawn_unified_scheduler(
         let mut address_book = solana_scheduler::AddressBook::default();
 
         solana_scheduler::ScheduleStage::run(
-            num_execution_threads * 100,
+            num_execution_threads * 10,
             &mut runnable_queue,
             &mut contended_queue,
             &mut address_book,

From 7d2130a78c96b79dbc131ca5b8b67e9fbbdea179 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 8 Aug 2022 09:50:14 +0900
Subject: [PATCH 0312/3199] save

---
 scheduler/src/lib.rs | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index d3af33b759fa2c..466b9a97d5db63 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -462,7 +462,7 @@ impl ScheduleStage {
     #[inline(never)]
     fn get_weight_from_contended(address_book: &AddressBook) -> Option<UniqueWeight> {
         let mut heaviest_weight: Option<UniqueWeight> = None;
-        error!("n u a len(): {}", address_book.newly_uncontended_addresses.len());
+        //error!("n u a len(): {}", address_book.newly_uncontended_addresses.len());
         for page in address_book.newly_uncontended_addresses.iter() {
             let newly_uncontended_unique_weights = &page.0.contended_unique_weights;
             if let Some(&weight) = newly_uncontended_unique_weights.last() {
@@ -600,9 +600,9 @@ impl ScheduleStage {
 
             // revert because now contended again
             if !from_runnable {
-                error!("n u a len() before: {}", address_book.newly_uncontended_addresses.len());
+                //error!("n u a len() before: {}", address_book.newly_uncontended_addresses.len());
                 address_book.newly_uncontended_addresses.remove(&l.status.page_rc().0);
-                error!("n u a len() after: {}", address_book.newly_uncontended_addresses.len());
+                //error!("n u a len() after: {}", address_book.newly_uncontended_addresses.len());
             }
 
             // todo: mem::forget and panic in LockAttempt::drop()

From 67161babb3dbd629d937103fb3e1a78446314044 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 8 Aug 2022 09:56:15 +0900
Subject: [PATCH 0313/3199] save

---
 ledger-tool/src/main.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ledger-tool/src/main.rs b/ledger-tool/src/main.rs
index bc34c0139a214f..af9ff8862ae6d8 100644
--- a/ledger-tool/src/main.rs
+++ b/ledger-tool/src/main.rs
@@ -280,7 +280,7 @@ fn output_slot(
                 &mut address_book,
                 &muxed_receiver,
                 &pre_execute_env_sender,
-                &post_schedule_env_sender,
+                //&post_schedule_env_sender,
             );
         })
         .unwrap();

From 092f2b2d78229fc9f176c9030d1bd13bf17b20f7 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 8 Aug 2022 09:56:58 +0900
Subject: [PATCH 0314/3199] save

---
 ledger-tool/src/main.rs                 | 2 +-
 scheduler/src/lib.rs                    | 4 ++--
 transaction-scheduler-bench/src/main.rs | 2 +-
 3 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/ledger-tool/src/main.rs b/ledger-tool/src/main.rs
index af9ff8862ae6d8..82bbf206b4857d 100644
--- a/ledger-tool/src/main.rs
+++ b/ledger-tool/src/main.rs
@@ -280,7 +280,7 @@ fn output_slot(
                 &mut address_book,
                 &muxed_receiver,
                 &pre_execute_env_sender,
-                //&post_schedule_env_sender,
+                Some(&post_schedule_env_sender),
             );
         })
         .unwrap();
diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 466b9a97d5db63..c753ab4a59a621 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -680,7 +680,7 @@ impl ScheduleStage {
         address_book: &mut AddressBook,
         from: &crossbeam_channel::Receiver<MultiplexedPayload>,
         to_execute_substage: &crossbeam_channel::Sender<Box<ExecutionEnvironment>>,
-        // to_next_stage: &crossbeam_channel::Sender<Box<ExecutionEnvironment>>, // assume nonblocking
+        to_next_stage: Option<&crossbeam_channel::Sender<Box<ExecutionEnvironment>>>, // assume nonblocking
     ) {
         let mut executing_queue_count = 0;
         let mut current_unique_key = u64::max_value();
@@ -721,7 +721,7 @@ impl ScheduleStage {
 
                     Self::commit_result(&mut processed_execution_environment, address_book);
                     // async-ly propagate the result to rpc subsystems
-                    // to_next_stage.send(processed_execution_environment).unwrap();
+                    to_next_stage.unwrap().send(processed_execution_environment).unwrap();
                 }
             }
 
diff --git a/transaction-scheduler-bench/src/main.rs b/transaction-scheduler-bench/src/main.rs
index d0933d65354692..3de261c828692b 100644
--- a/transaction-scheduler-bench/src/main.rs
+++ b/transaction-scheduler-bench/src/main.rs
@@ -137,7 +137,7 @@ fn spawn_unified_scheduler(
             &mut address_book,
             &packet_batch_receiver.clone(),
             &transaction_batch_senders[0],
-            //&completed_transaction_receiver
+            None,//&completed_transaction_receiver
         );
     }).unwrap()
 }

From 343c61be3f021699c0f52e84bfe9300efe663617 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 8 Aug 2022 10:06:34 +0900
Subject: [PATCH 0315/3199] save

---
 scheduler/src/lib.rs | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index c753ab4a59a621..daa82f7b8daacb 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -721,7 +721,9 @@ impl ScheduleStage {
 
                     Self::commit_result(&mut processed_execution_environment, address_book);
                     // async-ly propagate the result to rpc subsystems
-                    to_next_stage.unwrap().send(processed_execution_environment).unwrap();
+                    if let Some(to_next_stage) = to_next_stage {
+                        to_next_stage.send(processed_execution_environment).unwrap();
+                    }
                 }
             }
 

From 2f92d14993e7f1a335818c37162a9068a9b27e01 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 8 Aug 2022 10:11:24 +0900
Subject: [PATCH 0316/3199] save

---
 scheduler/src/lib.rs | 18 +++++++++---------
 1 file changed, 9 insertions(+), 9 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index daa82f7b8daacb..e408b3afa40ff8 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -19,9 +19,9 @@ use {
 
 type MyRcInner<T> = std::sync::Arc<T>;
 #[derive(Debug, Hash, PartialEq, Eq, Clone)]
-struct MyRc(ByAddress<MyRcInner<Page>>);
+struct PageRc(ByAddress<MyRcInner<Page>>);
 
-unsafe impl Send for MyRc {}
+unsafe impl Send for PageRc {}
 
 #[derive(Debug)]
 pub struct ExecutionEnvironment {
@@ -47,18 +47,18 @@ impl ExecutionEnvironment {
 #[derive(Clone, Debug)]
 enum LockAttemptStatus {
     BeforeLookup(Pubkey),
-    AfterLookup(MyRc),
+    AfterLookup(PageRc),
 }
 
 impl LockAttemptStatus {
-    fn page_rc(&mut self) -> &MyRc {
+    fn page_rc(&mut self) -> &PageRc {
         match self {
             LockAttemptStatus::BeforeLookup(_) => unreachable!(),
             LockAttemptStatus::AfterLookup(page) => page,
         }
     }
 
-    fn take_page_rc(mut self) -> MyRc {
+    fn take_page_rc(mut self) -> PageRc {
         match self {
             LockAttemptStatus::BeforeLookup(_) => unreachable!(),
             LockAttemptStatus::AfterLookup(page) => page,
@@ -144,7 +144,7 @@ impl Page {
     }
 }
 
-type AddressMap = std::collections::HashMap<Pubkey, MyRc>;
+type AddressMap = std::collections::HashMap<Pubkey, PageRc>;
 use by_address::ByAddress;
 type AddressSet = std::collections::HashSet<ByAddress<MyRcInner<Page>>>;
 type AddressMapEntry<'a, K, V> = std::collections::hash_map::Entry<'a, K, V>;
@@ -170,14 +170,14 @@ impl AddressBook {
             LockAttemptStatus::BeforeLookup(address) => {
                 match self.book.entry(*address) {
                     AddressMapEntry::Vacant(book_entry) => {
-                        let page = MyRc(ByAddress(MyRcInner::new(Page::new(CurrentUsage::renew(*requested_usage)))));
-                        *status = LockAttemptStatus::AfterLookup(MyRc::clone(&page));
+                        let page = PageRc(ByAddress(MyRcInner::new(Page::new(CurrentUsage::renew(*requested_usage)))));
+                        *status = LockAttemptStatus::AfterLookup(PageRc::clone(&page));
                         *is_success = true;
                         book_entry.insert(page);
                     }
                     AddressMapEntry::Occupied(mut book_entry) => {
                         let page = book_entry.get_mut();
-                        let cloned_page = MyRc::clone(&page);
+                        let cloned_page = PageRc::clone(&page);
                         //dbg!((&page, &cloned_page));
                         //assert_eq!(&*page, &cloned_page);
                         //let mut a = std::collections::HashSet::new();

From 4683a391834d3a77b49474762ec0aa2ec0bd0123 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 8 Aug 2022 10:13:02 +0900
Subject: [PATCH 0317/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index e408b3afa40ff8..de8b910bd7b6cc 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -146,7 +146,7 @@ impl Page {
 
 type AddressMap = std::collections::HashMap<Pubkey, PageRc>;
 use by_address::ByAddress;
-type AddressSet = std::collections::HashSet<ByAddress<MyRcInner<Page>>>;
+type AddressSet = std::collections::HashSet<PageRc>;
 type AddressMapEntry<'a, K, V> = std::collections::hash_map::Entry<'a, K, V>;
 
 // needs ttl mechanism and prune

From 30761e42728f6347e0d354e948bc34bc421a5508 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 8 Aug 2022 10:13:23 +0900
Subject: [PATCH 0318/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index de8b910bd7b6cc..3e765fa8dbeaac 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -585,7 +585,7 @@ impl ScheduleStage {
             address_book.forget_address_contention(&unique_weight, &mut l);
 
             // revert because now contended again
-            address_book.newly_uncontended_addresses.remove(&l.status.page_rc().0);
+            address_book.newly_uncontended_addresses.remove(&l.status.page_rc());
         }
     }
 

From e6240c486fa55cf3d8b0377786a68ca0e7451264 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 8 Aug 2022 10:13:33 +0900
Subject: [PATCH 0319/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 3e765fa8dbeaac..c16ccdb0502ec4 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -601,7 +601,7 @@ impl ScheduleStage {
             // revert because now contended again
             if !from_runnable {
                 //error!("n u a len() before: {}", address_book.newly_uncontended_addresses.len());
-                address_book.newly_uncontended_addresses.remove(&l.status.page_rc().0);
+                address_book.newly_uncontended_addresses.remove(l.status.page_rc());
                 //error!("n u a len() after: {}", address_book.newly_uncontended_addresses.len());
             }
 

From 1afb7b7e01a46d6fc9940718864facb3eb11920c Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 8 Aug 2022 10:13:47 +0900
Subject: [PATCH 0320/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index c16ccdb0502ec4..60985ddf44096e 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -601,7 +601,7 @@ impl ScheduleStage {
             // revert because now contended again
             if !from_runnable {
                 //error!("n u a len() before: {}", address_book.newly_uncontended_addresses.len());
-                address_book.newly_uncontended_addresses.remove(l.status.page_rc());
+                address_book.newly_uncontended_addresses.remove(&l.status.page_rc());
                 //error!("n u a len() after: {}", address_book.newly_uncontended_addresses.len());
             }
 

From f85ffdeb8f8334f3d675b9eb1e73ff6ed9943e8f Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 8 Aug 2022 10:14:00 +0900
Subject: [PATCH 0321/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 60985ddf44096e..b9566bc1e5bf9a 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -614,7 +614,7 @@ impl ScheduleStage {
         for mut l in lock_attempts.into_iter() {
             let newly_uncontended_while_queued = address_book.unlock(&mut l);
             if newly_uncontended_while_queued {
-                address_book.newly_uncontended_addresses.insert(l.status.take_page_rc().0);
+                address_book.newly_uncontended_addresses.insert(l.status.take_page_rc());
             }
 
             // todo: mem::forget and panic in LockAttempt::drop()

From 952e4e014eceffd946e6ef9214092fbc8ef82980 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 8 Aug 2022 10:18:54 +0900
Subject: [PATCH 0322/3199] save

---
 scheduler/src/lib.rs | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index b9566bc1e5bf9a..6ae173906dce56 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -17,7 +17,8 @@ use {
     },
 };
 
-type MyRcInner<T> = std::sync::Arc<T>;
+type MyRcInner<T> = std::rc::Rc<T>;
+//type MyRcInner<T> = std::sync::Arc<T>;
 #[derive(Debug, Hash, PartialEq, Eq, Clone)]
 struct PageRc(ByAddress<MyRcInner<Page>>);
 

From 83e4823688c2ca32f057a4269b05c6416b03832d Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 8 Aug 2022 10:21:59 +0900
Subject: [PATCH 0323/3199] save

---
 scheduler/src/lib.rs | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 6ae173906dce56..8e70efaf5c3f37 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -17,12 +17,16 @@ use {
     },
 };
 
+/*
 type MyRcInner<T> = std::rc::Rc<T>;
-//type MyRcInner<T> = std::sync::Arc<T>;
+unsafe impl Send for PageRc {}
+*/
+
+type MyRcInner<T> = std::sync::Arc<T>;
+
 #[derive(Debug, Hash, PartialEq, Eq, Clone)]
 struct PageRc(ByAddress<MyRcInner<Page>>);
 
-unsafe impl Send for PageRc {}
 
 #[derive(Debug)]
 pub struct ExecutionEnvironment {

From 4789a701a9dc4c72a8049203e5a01fef831c24df Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 8 Aug 2022 10:22:55 +0900
Subject: [PATCH 0324/3199] save

---
 scheduler/src/lib.rs | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 8e70efaf5c3f37..549d2220138ae8 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -149,7 +149,8 @@ impl Page {
     }
 }
 
-type AddressMap = std::collections::HashMap<Pubkey, PageRc>;
+//type AddressMap = std::collections::HashMap<Pubkey, PageRc>;
+type AddressMap = dashmap::DashMap<Pubkey, PageRc>;
 use by_address::ByAddress;
 type AddressSet = std::collections::HashSet<PageRc>;
 type AddressMapEntry<'a, K, V> = std::collections::hash_map::Entry<'a, K, V>;

From fe7c8c4047d083c7b2eb42a0bcd81e93ab0a64e6 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 8 Aug 2022 10:24:04 +0900
Subject: [PATCH 0325/3199] save

---
 scheduler/Cargo.toml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/scheduler/Cargo.toml b/scheduler/Cargo.toml
index 3935355d558249..bb159ba423148c 100644
--- a/scheduler/Cargo.toml
+++ b/scheduler/Cargo.toml
@@ -14,6 +14,7 @@ publish = true
 atomic_enum = "0.1.1"
 by_address = "1.0.4"
 crossbeam-channel = "0.5.5"
+dashmap = { version = "4.0.2" }
 log = "0.4.17"
 solana-entry = { path = "../entry", version = "=1.11.5" }
 solana-runtime = { path = "../runtime", version = "=1.11.5" }

From 42a8fef0fadd69f7722fc8adc5b3f37cac178849 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 8 Aug 2022 10:24:55 +0900
Subject: [PATCH 0326/3199] save

---
 Cargo.lock           | 1 +
 scheduler/src/lib.rs | 3 ++-
 2 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/Cargo.lock b/Cargo.lock
index 1626ee9ab5d8d3..1846dcf5313025 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -6075,6 +6075,7 @@ dependencies = [
  "atomic_enum",
  "by_address",
  "crossbeam-channel",
+ "dashmap",
  "log",
  "rand 0.7.3",
  "sha2 0.10.2",
diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 549d2220138ae8..a542aa8dd9567f 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -153,7 +153,8 @@ impl Page {
 type AddressMap = dashmap::DashMap<Pubkey, PageRc>;
 use by_address::ByAddress;
 type AddressSet = std::collections::HashSet<PageRc>;
-type AddressMapEntry<'a, K, V> = std::collections::hash_map::Entry<'a, K, V>;
+//type AddressMapEntry<'a, K, V> = std::collections::hash_map::Entry<'a, K, V>;
+type AddressMapEntry<'a> = dashmap::mapref::entry::Entry<'a, Pubkey, PageRc>;
 
 // needs ttl mechanism and prune
 #[derive(Default)]

From dda724517a655230a73ec589eac561915a3427d4 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 8 Aug 2022 10:25:26 +0900
Subject: [PATCH 0327/3199] save

---
 scheduler/src/lib.rs | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index a542aa8dd9567f..d498c9d563a02d 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -453,6 +453,7 @@ impl ScheduleStage {
         *unique_key -= 1;
     }
 
+    /*
     #[inline(never)]
     fn get_newly_u_u_w<'a>(
         address: &'a Pubkey,
@@ -465,6 +466,7 @@ impl ScheduleStage {
             .0
             .contended_unique_weights
     }
+    */
 
     #[inline(never)]
     fn get_weight_from_contended(address_book: &AddressBook) -> Option<UniqueWeight> {

From bd51d134ee6cccc3a4d5e82ecaa3e0f4b62a7a4a Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 8 Aug 2022 10:28:56 +0900
Subject: [PATCH 0328/3199] save

---
 scheduler/src/lib.rs | 32 ++++++++++++++++----------------
 1 file changed, 16 insertions(+), 16 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index d498c9d563a02d..ac47c9562396a8 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -50,37 +50,37 @@ impl ExecutionEnvironment {
 }
 
 #[derive(Clone, Debug)]
-enum LockAttemptStatus {
-    BeforeLookup(Pubkey),
-    AfterLookup(PageRc),
+enum AddressLookup {
+    Before(Pubkey),
+    After(PageRc),
 }
 
-impl LockAttemptStatus {
+impl AddressLookup {
     fn page_rc(&mut self) -> &PageRc {
         match self {
-            LockAttemptStatus::BeforeLookup(_) => unreachable!(),
-            LockAttemptStatus::AfterLookup(page) => page,
+            AddressLookup::Before(_) => unreachable!(),
+            AddressLookup::After(page) => page,
         }
     }
 
     fn take_page_rc(mut self) -> PageRc {
         match self {
-            LockAttemptStatus::BeforeLookup(_) => unreachable!(),
-            LockAttemptStatus::AfterLookup(page) => page,
+            AddressLookup::Before(_) => unreachable!(),
+            AddressLookup::After(page) => page,
         }
     }
 
     fn page(&mut self) -> &mut Page {
         match self {
-            LockAttemptStatus::BeforeLookup(_) => unreachable!(),
-            LockAttemptStatus::AfterLookup(page) => unsafe { MyRcInner::get_mut_unchecked(&mut page.0) },
+            AddressLookup::Before(_) => unreachable!(),
+            AddressLookup::After(page) => unsafe { MyRcInner::get_mut_unchecked(&mut page.0) },
         }
     }
 }
 
 #[derive(Clone, Debug)]
 pub struct LockAttempt {
-    status: LockAttemptStatus,
+    status: AddressLookup,
     is_success: bool,
     requested_usage: RequestedUsage,
 }
@@ -96,7 +96,7 @@ impl LockAttempt {
 
     pub fn new(address: Pubkey, requested_usage: RequestedUsage) -> Self {
         Self {
-            status: LockAttemptStatus::BeforeLookup(address),
+            status: AddressLookup::Before(address),
             is_success: true,
             requested_usage,
         }
@@ -174,11 +174,11 @@ impl AddressBook {
         let LockAttempt {status, requested_usage, is_success} = attempt;
 
         match status {
-            LockAttemptStatus::BeforeLookup(address) => {
+            AddressLookup::Before(address) => {
                 match self.book.entry(*address) {
                     AddressMapEntry::Vacant(book_entry) => {
                         let page = PageRc(ByAddress(MyRcInner::new(Page::new(CurrentUsage::renew(*requested_usage)))));
-                        *status = LockAttemptStatus::AfterLookup(PageRc::clone(&page));
+                        *status = AddressLookup::After(PageRc::clone(&page));
                         *is_success = true;
                         book_entry.insert(page);
                     }
@@ -191,7 +191,7 @@ impl AddressBook {
                         //a.insert(page.clone());
                         //a.insert(cloned_page);
                         //panic!("{:?}", a);
-                        *status = LockAttemptStatus::AfterLookup(cloned_page);
+                        *status = AddressLookup::After(cloned_page);
                         let mut page = unsafe { MyRcInner::get_mut_unchecked(&mut page.0) };
 
                         match page.current_usage {
@@ -223,7 +223,7 @@ impl AddressBook {
                     }
                 }
             }
-            LockAttemptStatus::AfterLookup(page) => {
+            AddressLookup::After(page) => {
                 let mut page = unsafe { MyRcInner::get_mut_unchecked(&mut page.0) };
 
                 match page.current_usage {

From 1713c60113c49253f88e7265279b83a15b8b8b5f Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 8 Aug 2022 10:31:05 +0900
Subject: [PATCH 0329/3199] save

---
 scheduler/src/lib.rs | 22 +++++++++++-----------
 1 file changed, 11 insertions(+), 11 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index ac47c9562396a8..c607194d1b6e86 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -80,7 +80,7 @@ impl AddressLookup {
 
 #[derive(Clone, Debug)]
 pub struct LockAttempt {
-    status: AddressLookup,
+    lookup: AddressLookup,
     is_success: bool,
     requested_usage: RequestedUsage,
 }
@@ -96,7 +96,7 @@ impl LockAttempt {
 
     pub fn new(address: Pubkey, requested_usage: RequestedUsage) -> Self {
         Self {
-            status: AddressLookup::Before(address),
+            lookup: AddressLookup::Before(address),
             is_success: true,
             requested_usage,
         }
@@ -171,14 +171,14 @@ impl AddressBook {
         unique_weight: &UniqueWeight,
         attempt: &mut LockAttempt,
     ) {
-        let LockAttempt {status, requested_usage, is_success} = attempt;
+        let LockAttempt {lookup, requested_usage, is_success} = attempt;
 
-        match status {
+        match lookup {
             AddressLookup::Before(address) => {
                 match self.book.entry(*address) {
                     AddressMapEntry::Vacant(book_entry) => {
                         let page = PageRc(ByAddress(MyRcInner::new(Page::new(CurrentUsage::renew(*requested_usage)))));
-                        *status = AddressLookup::After(PageRc::clone(&page));
+                        *lookup = AddressLookup::After(PageRc::clone(&page));
                         *is_success = true;
                         book_entry.insert(page);
                     }
@@ -191,7 +191,7 @@ impl AddressBook {
                         //a.insert(page.clone());
                         //a.insert(cloned_page);
                         //panic!("{:?}", a);
-                        *status = AddressLookup::After(cloned_page);
+                        *lookup = AddressLookup::After(cloned_page);
                         let mut page = unsafe { MyRcInner::get_mut_unchecked(&mut page.0) };
 
                         match page.current_usage {
@@ -263,7 +263,7 @@ impl AddressBook {
 
     #[inline(never)]
     fn forget_address_contention(&mut self, unique_weight: &UniqueWeight, a: &mut LockAttempt) {
-        a.status.page().contended_unique_weights.remove(unique_weight);
+        a.lookup.page().contended_unique_weights.remove(unique_weight);
     }
 
     fn ensure_unlock(&mut self, attempt: &mut LockAttempt) {
@@ -279,7 +279,7 @@ impl AddressBook {
         let mut newly_uncontended = false;
         let mut still_queued = false;
 
-        let mut page = attempt.status.page();
+        let mut page = attempt.lookup.page();
 
         match &mut page.current_usage {
             CurrentUsage::Readonly(ref mut count) => match &attempt.requested_usage {
@@ -594,7 +594,7 @@ impl ScheduleStage {
             address_book.forget_address_contention(&unique_weight, &mut l);
 
             // revert because now contended again
-            address_book.newly_uncontended_addresses.remove(&l.status.page_rc());
+            address_book.newly_uncontended_addresses.remove(&l.lookup.page_rc());
         }
     }
 
@@ -610,7 +610,7 @@ impl ScheduleStage {
             // revert because now contended again
             if !from_runnable {
                 //error!("n u a len() before: {}", address_book.newly_uncontended_addresses.len());
-                address_book.newly_uncontended_addresses.remove(&l.status.page_rc());
+                address_book.newly_uncontended_addresses.remove(&l.lookup.page_rc());
                 //error!("n u a len() after: {}", address_book.newly_uncontended_addresses.len());
             }
 
@@ -623,7 +623,7 @@ impl ScheduleStage {
         for mut l in lock_attempts.into_iter() {
             let newly_uncontended_while_queued = address_book.unlock(&mut l);
             if newly_uncontended_while_queued {
-                address_book.newly_uncontended_addresses.insert(l.status.take_page_rc());
+                address_book.newly_uncontended_addresses.insert(l.lookup.take_page_rc());
             }
 
             // todo: mem::forget and panic in LockAttempt::drop()

From 6ecbb93f19f8f1201e189bea7117c8e5329b93bc Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 8 Aug 2022 10:33:35 +0900
Subject: [PATCH 0330/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index c607194d1b6e86..5c3ae70f3dd669 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -80,7 +80,7 @@ impl AddressLookup {
 
 #[derive(Clone, Debug)]
 pub struct LockAttempt {
-    lookup: AddressLookup,
+    target: PageRc,
     is_success: bool,
     requested_usage: RequestedUsage,
 }

From 4601064e0bf0d3ebcba47723988062bb5c88e529 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 8 Aug 2022 10:34:02 +0900
Subject: [PATCH 0331/3199] save

---
 scheduler/src/lib.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 5c3ae70f3dd669..f7cae6ab6fa291 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -94,9 +94,9 @@ impl LockAttempt {
         !self.is_success()
     }
 
-    pub fn new(address: Pubkey, requested_usage: RequestedUsage) -> Self {
+    pub fn new(targe: PageRc, requested_usage: RequestedUsage) -> Self {
         Self {
-            lookup: AddressLookup::Before(address),
+            target,
             is_success: true,
             requested_usage,
         }

From 07cd5314d87f52b468b800f00e4f0f6c2c12dbdc Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 8 Aug 2022 10:34:54 +0900
Subject: [PATCH 0332/3199] save

---
 scheduler/src/lib.rs | 59 +++-----------------------------------------
 1 file changed, 3 insertions(+), 56 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index f7cae6ab6fa291..a202215f35c07c 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -171,60 +171,9 @@ impl AddressBook {
         unique_weight: &UniqueWeight,
         attempt: &mut LockAttempt,
     ) {
-        let LockAttempt {lookup, requested_usage, is_success} = attempt;
-
-        match lookup {
-            AddressLookup::Before(address) => {
-                match self.book.entry(*address) {
-                    AddressMapEntry::Vacant(book_entry) => {
-                        let page = PageRc(ByAddress(MyRcInner::new(Page::new(CurrentUsage::renew(*requested_usage)))));
-                        *lookup = AddressLookup::After(PageRc::clone(&page));
-                        *is_success = true;
-                        book_entry.insert(page);
-                    }
-                    AddressMapEntry::Occupied(mut book_entry) => {
-                        let page = book_entry.get_mut();
-                        let cloned_page = PageRc::clone(&page);
-                        //dbg!((&page, &cloned_page));
-                        //assert_eq!(&*page, &cloned_page);
-                        //let mut a = std::collections::HashSet::new();
-                        //a.insert(page.clone());
-                        //a.insert(cloned_page);
-                        //panic!("{:?}", a);
-                        *lookup = AddressLookup::After(cloned_page);
-                        let mut page = unsafe { MyRcInner::get_mut_unchecked(&mut page.0) };
-
-                        match page.current_usage {
-                            CurrentUsage::Unused => {
-                                page.current_usage = CurrentUsage::renew(*requested_usage);
-                                *is_success = true;
-                            }
-                            CurrentUsage::Readonly(ref mut count) => match requested_usage {
-                                RequestedUsage::Readonly => {
-                                    *count += 1;
-                                    *is_success = true;
-                                }
-                                RequestedUsage::Writable => {
-                                    if from_runnable {
-                                        Self::remember_address_contention(&mut page, unique_weight);
-                                    }
-                                    *is_success = false;
-                                }
-                            },
-                            CurrentUsage::Writable => match requested_usage {
-                                RequestedUsage::Readonly | RequestedUsage::Writable => {
-                                    if from_runnable {
-                                        Self::remember_address_contention(&mut page, unique_weight);
-                                    }
-                                    *is_success = false;
-                                }
-                            },
-                        }
-                    }
-                }
-            }
-            AddressLookup::After(page) => {
-                let mut page = unsafe { MyRcInner::get_mut_unchecked(&mut page.0) };
+        let LockAttempt {target, requested_usage, is_success} = attempt;
+
+                let mut page = unsafe { MyRcInner::get_mut_unchecked(&mut target.0) };
 
                 match page.current_usage {
                     CurrentUsage::Unused => {
@@ -252,8 +201,6 @@ impl AddressBook {
                         }
                     },
                 }
-            }
-        }
     }
 
     #[inline(never)]

From 93e938e19c1962abb055b47e36f6eada14abf56e Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 8 Aug 2022 10:35:17 +0900
Subject: [PATCH 0333/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index a202215f35c07c..3a691bcea743bd 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -210,7 +210,7 @@ impl AddressBook {
 
     #[inline(never)]
     fn forget_address_contention(&mut self, unique_weight: &UniqueWeight, a: &mut LockAttempt) {
-        a.lookup.page().contended_unique_weights.remove(unique_weight);
+        a.target.contended_unique_weights.remove(unique_weight);
     }
 
     fn ensure_unlock(&mut self, attempt: &mut LockAttempt) {

From 86327c8730c75fb95685a1cc629f40466f4e9ee9 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 8 Aug 2022 10:35:58 +0900
Subject: [PATCH 0334/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 3a691bcea743bd..b9513dfaf71731 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -210,7 +210,7 @@ impl AddressBook {
 
     #[inline(never)]
     fn forget_address_contention(&mut self, unique_weight: &UniqueWeight, a: &mut LockAttempt) {
-        a.target.contended_unique_weights.remove(unique_weight);
+        a.target.0.contended_unique_weights.remove(unique_weight);
     }
 
     fn ensure_unlock(&mut self, attempt: &mut LockAttempt) {

From 697202f885a77a9847e9e8775ef3e210deb04d20 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 8 Aug 2022 10:36:21 +0900
Subject: [PATCH 0335/3199] save

---
 scheduler/src/lib.rs | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index b9513dfaf71731..1bc421a31a5648 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -226,7 +226,7 @@ impl AddressBook {
         let mut newly_uncontended = false;
         let mut still_queued = false;
 
-        let mut page = attempt.lookup.page();
+        let mut page = attempt.target.page();
 
         match &mut page.current_usage {
             CurrentUsage::Readonly(ref mut count) => match &attempt.requested_usage {
@@ -541,7 +541,7 @@ impl ScheduleStage {
             address_book.forget_address_contention(&unique_weight, &mut l);
 
             // revert because now contended again
-            address_book.newly_uncontended_addresses.remove(&l.lookup.page_rc());
+            address_book.newly_uncontended_addresses.remove(&l.target.page_rc());
         }
     }
 
@@ -557,7 +557,7 @@ impl ScheduleStage {
             // revert because now contended again
             if !from_runnable {
                 //error!("n u a len() before: {}", address_book.newly_uncontended_addresses.len());
-                address_book.newly_uncontended_addresses.remove(&l.lookup.page_rc());
+                address_book.newly_uncontended_addresses.remove(&l.target.page_rc());
                 //error!("n u a len() after: {}", address_book.newly_uncontended_addresses.len());
             }
 

From 008b57951afd3b097f46438a64cb4ecd955b8786 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 8 Aug 2022 10:36:32 +0900
Subject: [PATCH 0336/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 1bc421a31a5648..1885c9855f3c43 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -570,7 +570,7 @@ impl ScheduleStage {
         for mut l in lock_attempts.into_iter() {
             let newly_uncontended_while_queued = address_book.unlock(&mut l);
             if newly_uncontended_while_queued {
-                address_book.newly_uncontended_addresses.insert(l.lookup.take_page_rc());
+                address_book.newly_uncontended_addresses.insert(l.target.take_page_rc());
             }
 
             // todo: mem::forget and panic in LockAttempt::drop()

From 70def9a84f5eb2ab96fb8b71aa6daf4b06a28603 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 8 Aug 2022 10:36:40 +0900
Subject: [PATCH 0337/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 1885c9855f3c43..62d2e15b34eda3 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -570,7 +570,7 @@ impl ScheduleStage {
         for mut l in lock_attempts.into_iter() {
             let newly_uncontended_while_queued = address_book.unlock(&mut l);
             if newly_uncontended_while_queued {
-                address_book.newly_uncontended_addresses.insert(l.target.take_page_rc());
+                address_book.newly_uncontended_addresses.insert(l.target);
             }
 
             // todo: mem::forget and panic in LockAttempt::drop()

From 10846ef784fdd707ba8832aab72b9f37f919cee4 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 8 Aug 2022 10:38:28 +0900
Subject: [PATCH 0338/3199] save

---
 scheduler/src/lib.rs | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 62d2e15b34eda3..c6a95089d81d3b 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -78,6 +78,12 @@ impl AddressLookup {
     }
 }
 
+impl PageRc {
+    fn page(&mut self) -> &mut Page {
+        MyRcInner::get_mut_unchecked(&mut page.0)
+    }
+}
+
 #[derive(Clone, Debug)]
 pub struct LockAttempt {
     target: PageRc,
@@ -541,7 +547,7 @@ impl ScheduleStage {
             address_book.forget_address_contention(&unique_weight, &mut l);
 
             // revert because now contended again
-            address_book.newly_uncontended_addresses.remove(&l.target.page_rc());
+            address_book.newly_uncontended_addresses.remove(&l.target);
         }
     }
 
@@ -557,7 +563,7 @@ impl ScheduleStage {
             // revert because now contended again
             if !from_runnable {
                 //error!("n u a len() before: {}", address_book.newly_uncontended_addresses.len());
-                address_book.newly_uncontended_addresses.remove(&l.target.page_rc());
+                address_book.newly_uncontended_addresses.remove(&l.target);
                 //error!("n u a len() after: {}", address_book.newly_uncontended_addresses.len());
             }
 

From d0136b74d1a9519e43e814dc4fcef457be7e5f1a Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 8 Aug 2022 10:38:41 +0900
Subject: [PATCH 0339/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index c6a95089d81d3b..68c49b7d76afd7 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -80,7 +80,7 @@ impl AddressLookup {
 
 impl PageRc {
     fn page(&mut self) -> &mut Page {
-        MyRcInner::get_mut_unchecked(&mut page.0)
+        MyRcInner::get_mut_unchecked(&mut self.0)
     }
 }
 

From eddf2946dda7d934b970a55c6fc815d38bf60db5 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 8 Aug 2022 10:38:52 +0900
Subject: [PATCH 0340/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 68c49b7d76afd7..be02762b1b0feb 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -100,7 +100,7 @@ impl LockAttempt {
         !self.is_success()
     }
 
-    pub fn new(targe: PageRc, requested_usage: RequestedUsage) -> Self {
+    pub fn new(target: PageRc, requested_usage: RequestedUsage) -> Self {
         Self {
             target,
             is_success: true,

From ed96f813950df5a1e311ed319aaf5857d6d73c97 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 8 Aug 2022 10:39:06 +0900
Subject: [PATCH 0341/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index be02762b1b0feb..8598049791188e 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -216,7 +216,7 @@ impl AddressBook {
 
     #[inline(never)]
     fn forget_address_contention(&mut self, unique_weight: &UniqueWeight, a: &mut LockAttempt) {
-        a.target.0.contended_unique_weights.remove(unique_weight);
+        a.target.page().contended_unique_weights.remove(unique_weight);
     }
 
     fn ensure_unlock(&mut self, attempt: &mut LockAttempt) {

From 3d1d444f424530bf0e86d648c5dda3fd4690351b Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 8 Aug 2022 10:39:18 +0900
Subject: [PATCH 0342/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 8598049791188e..8710f88a064cb8 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -80,7 +80,7 @@ impl AddressLookup {
 
 impl PageRc {
     fn page(&mut self) -> &mut Page {
-        MyRcInner::get_mut_unchecked(&mut self.0)
+        unsafe { MyRcInner::get_mut_unchecked(&mut self.0) }
     }
 }
 

From 0c3c42dc707d6ab6294b9b97fae1f3b3e46a1ff0 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 8 Aug 2022 10:39:35 +0900
Subject: [PATCH 0343/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 8710f88a064cb8..beba515585a70c 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -25,7 +25,7 @@ unsafe impl Send for PageRc {}
 type MyRcInner<T> = std::sync::Arc<T>;
 
 #[derive(Debug, Hash, PartialEq, Eq, Clone)]
-struct PageRc(ByAddress<MyRcInner<Page>>);
+pub struct PageRc(ByAddress<MyRcInner<Page>>);
 
 
 #[derive(Debug)]

From 4c48438d79223e9df9428b5040141ec3b39b380a Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 8 Aug 2022 10:44:16 +0900
Subject: [PATCH 0344/3199] save

---
 ledger-tool/src/main.rs |  1 +
 scheduler/src/lib.rs    | 10 +++++++++-
 2 files changed, 10 insertions(+), 1 deletion(-)

diff --git a/ledger-tool/src/main.rs b/ledger-tool/src/main.rs
index 82bbf206b4857d..bb44b2673a548f 100644
--- a/ledger-tool/src/main.rs
+++ b/ledger-tool/src/main.rs
@@ -359,6 +359,7 @@ fn output_slot(
     if verbose_level >= 2 {
         let mut txes = Vec::new();
         let skip_voting = std::env::var("SKIP_VOTING").is_ok();
+        let preloader = address_book.preloader();
         for (entry_index, entry) in entries.into_iter().enumerate() {
             output_entry(
                 blockstore,
diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index beba515585a70c..0e9cf8026950b4 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -156,7 +156,7 @@ impl Page {
 }
 
 //type AddressMap = std::collections::HashMap<Pubkey, PageRc>;
-type AddressMap = dashmap::DashMap<Pubkey, PageRc>;
+type AddressMap = std::sync::Arc<dashmap::DashMap<Pubkey, PageRc>>;
 use by_address::ByAddress;
 type AddressSet = std::collections::HashSet<PageRc>;
 //type AddressMapEntry<'a, K, V> = std::collections::hash_map::Entry<'a, K, V>;
@@ -263,6 +263,14 @@ impl AddressBook {
 
         still_queued
     }
+
+    fn preloader() -> Preloader {
+        Preloader{map: Arc::clone(self.map)}
+    }
+}
+
+pub struct Preloader {
+    map: AddressMap,
 }
 
 #[derive(Copy, Clone, Debug, PartialEq, Eq, PartialOrd, Ord)]

From fd175234b06047161aea7870e7921d2d0919b475 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 8 Aug 2022 10:44:27 +0900
Subject: [PATCH 0345/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 0e9cf8026950b4..1a5687d73004b9 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -264,7 +264,7 @@ impl AddressBook {
         still_queued
     }
 
-    fn preloader() -> Preloader {
+    fn preloader(&self) -> Preloader {
         Preloader{map: Arc::clone(self.map)}
     }
 }

From e18a617a8e754dc345657e63ff08cc800a959f46 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 8 Aug 2022 10:44:34 +0900
Subject: [PATCH 0346/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 1a5687d73004b9..85764bc7fdf508 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -265,7 +265,7 @@ impl AddressBook {
     }
 
     fn preloader(&self) -> Preloader {
-        Preloader{map: Arc::clone(self.map)}
+        Preloader{map: Arc::clone(self.book)}
     }
 }
 

From 56de1c23af618e3a10b073d0529ad07f5f69223f Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 8 Aug 2022 10:44:45 +0900
Subject: [PATCH 0347/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 85764bc7fdf508..834c919599c6d2 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -265,7 +265,7 @@ impl AddressBook {
     }
 
     fn preloader(&self) -> Preloader {
-        Preloader{map: Arc::clone(self.book)}
+        Preloader{map: std::sync::Arc::clone(self.book)}
     }
 }
 

From ac257a4c4ad587a501652e8f846bc65966feb1d3 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 8 Aug 2022 10:44:55 +0900
Subject: [PATCH 0348/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 834c919599c6d2..9c2bcfc6f555b9 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -265,7 +265,7 @@ impl AddressBook {
     }
 
     fn preloader(&self) -> Preloader {
-        Preloader{map: std::sync::Arc::clone(self.book)}
+        Preloader{map: std::sync::Arc::clone(&self.book)}
     }
 }
 

From 483f284449d1e8dbf78f1b1012175a8defea979b Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 8 Aug 2022 10:45:25 +0900
Subject: [PATCH 0349/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 9c2bcfc6f555b9..dce16e6203beb6 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -264,7 +264,7 @@ impl AddressBook {
         still_queued
     }
 
-    fn preloader(&self) -> Preloader {
+    pub fn preloader(&self) -> Preloader {
         Preloader{map: std::sync::Arc::clone(&self.book)}
     }
 }

From 5643a959142a0833d04ee39b922d6995da6bc217 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 8 Aug 2022 10:46:00 +0900
Subject: [PATCH 0350/3199] save

---
 ledger-tool/src/main.rs | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/ledger-tool/src/main.rs b/ledger-tool/src/main.rs
index bb44b2673a548f..49db937522f512 100644
--- a/ledger-tool/src/main.rs
+++ b/ledger-tool/src/main.rs
@@ -141,6 +141,7 @@ fn output_entry(
     slot: Slot,
     entry_index: usize,
     entry: Entry,
+    preloader: &Preloader,
     to_schedule_stage: &mut Vec<Box<(SanitizedTransaction, Vec<LockAttempt>)>>,
     skip_voting: bool,
 ) {
@@ -367,6 +368,7 @@ fn output_slot(
                 slot,
                 entry_index,
                 entry,
+                preloader,
                 &mut txes,
                 skip_voting,
             );

From a76d09d801dc3ee74e5c64dae552261adfcae69d Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 8 Aug 2022 10:46:11 +0900
Subject: [PATCH 0351/3199] save

---
 ledger-tool/src/main.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ledger-tool/src/main.rs b/ledger-tool/src/main.rs
index 49db937522f512..073a7d430d1197 100644
--- a/ledger-tool/src/main.rs
+++ b/ledger-tool/src/main.rs
@@ -52,7 +52,7 @@ use {
             DEFAULT_MAX_INCREMENTAL_SNAPSHOT_ARCHIVES_TO_RETAIN, SUPPORTED_ARCHIVE_COMPRESSION,
         },
     },
-    solana_scheduler::{AddressBook, ScheduleStage, TaskQueue, Weight, LockAttempt, RequestedUsage},
+    solana_scheduler::{AddressBook, ScheduleStage, TaskQueue, Weight, LockAttempt, RequestedUsage, Preloader},
     solana_sdk::{
         account::{AccountSharedData, ReadableAccount, WritableAccount},
         account_utils::StateMut,

From 2a1c0db7f35e7ce860528323ca230ba2b87b27f5 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 8 Aug 2022 10:46:29 +0900
Subject: [PATCH 0352/3199] save

---
 ledger-tool/src/main.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ledger-tool/src/main.rs b/ledger-tool/src/main.rs
index 073a7d430d1197..48ac0d1d2ca17c 100644
--- a/ledger-tool/src/main.rs
+++ b/ledger-tool/src/main.rs
@@ -141,7 +141,7 @@ fn output_entry(
     slot: Slot,
     entry_index: usize,
     entry: Entry,
-    preloader: &Preloader,
+    preloader: Preloader,
     to_schedule_stage: &mut Vec<Box<(SanitizedTransaction, Vec<LockAttempt>)>>,
     skip_voting: bool,
 ) {

From e53eb8f6ef2be7b37045a8aff88473c7ebd8e558 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 8 Aug 2022 10:47:49 +0900
Subject: [PATCH 0353/3199] save

---
 ledger-tool/src/main.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/ledger-tool/src/main.rs b/ledger-tool/src/main.rs
index 48ac0d1d2ca17c..ad8a92979c02fc 100644
--- a/ledger-tool/src/main.rs
+++ b/ledger-tool/src/main.rs
@@ -173,11 +173,11 @@ fn output_entry(
                     let writable_lock_iter = locks
                         .writable
                         .iter()
-                        .map(|address| LockAttempt::new(**address, RequestedUsage::Writable));
+                        .map(|address| LockAttempt::new(preloader.load(**address), RequestedUsage::Writable));
                     let readonly_lock_iter = locks
                         .readonly
                         .iter()
-                        .map(|address| LockAttempt::new(**address, RequestedUsage::Readonly));
+                        .map(|address| LockAttempt::new(preloader.load(**address), RequestedUsage::Readonly));
                     let locks = writable_lock_iter.chain(readonly_lock_iter).collect::<Vec<_>>();
                     to_schedule_stage.push(Box::new((sanitized_tx, locks)));
                 }

From af928e6830f4f42432d3add8bba76e250f468d09 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 8 Aug 2022 10:48:19 +0900
Subject: [PATCH 0354/3199] save

---
 scheduler/src/lib.rs | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index dce16e6203beb6..623d87287c3e69 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -273,6 +273,11 @@ pub struct Preloader {
     map: AddressMap,
 }
 
+impl Preloader {
+    fn load(address: Pubkey) -> PageRc {
+    }
+}
+
 #[derive(Copy, Clone, Debug, PartialEq, Eq, PartialOrd, Ord)]
 pub struct Weight {
     // naming: Sequence Ordering?

From d07515266a35968768da1f5c0dec1ac6b2d8b3f8 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 8 Aug 2022 10:48:30 +0900
Subject: [PATCH 0355/3199] save

---
 scheduler/src/lib.rs | 1 +
 1 file changed, 1 insertion(+)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 623d87287c3e69..23e4c0a7ccb45e 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -275,6 +275,7 @@ pub struct Preloader {
 
 impl Preloader {
     fn load(address: Pubkey) -> PageRc {
+        todo!()
     }
 }
 

From 8110b222deeb846a19109d3a385be58d49973ac8 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 8 Aug 2022 10:48:42 +0900
Subject: [PATCH 0356/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 23e4c0a7ccb45e..3423ea8d3db309 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -274,7 +274,7 @@ pub struct Preloader {
 }
 
 impl Preloader {
-    fn load(address: Pubkey) -> PageRc {
+    fn load(&self, address: Pubkey) -> PageRc {
         todo!()
     }
 }

From b3bb6cd4b5243efe233aaf95eed9c65f750602d7 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 8 Aug 2022 10:48:52 +0900
Subject: [PATCH 0357/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 3423ea8d3db309..cc550bc02e6f79 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -274,7 +274,7 @@ pub struct Preloader {
 }
 
 impl Preloader {
-    fn load(&self, address: Pubkey) -> PageRc {
+    pub fn load(&self, address: Pubkey) -> PageRc {
         todo!()
     }
 }

From 83c86c6c2a4714bf0b7020ec4bfd3832f339aea9 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 8 Aug 2022 10:49:20 +0900
Subject: [PATCH 0358/3199] save

---
 ledger-tool/src/main.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ledger-tool/src/main.rs b/ledger-tool/src/main.rs
index ad8a92979c02fc..f1eaea430fbe71 100644
--- a/ledger-tool/src/main.rs
+++ b/ledger-tool/src/main.rs
@@ -271,6 +271,7 @@ fn output_slot(
     let mut runnable_queue = TaskQueue::default();
     let mut contended_queue = TaskQueue::default();
     let mut address_book = AddressBook::default();
+    let preloader = address_book.preloader();
     let t1 = std::thread::Builder::new()
         .name("sol-scheduler".to_string())
         .spawn(move || loop {
@@ -360,7 +361,6 @@ fn output_slot(
     if verbose_level >= 2 {
         let mut txes = Vec::new();
         let skip_voting = std::env::var("SKIP_VOTING").is_ok();
-        let preloader = address_book.preloader();
         for (entry_index, entry) in entries.into_iter().enumerate() {
             output_entry(
                 blockstore,

From 23f60f2eb52f7cb62b55c16ecae6c90ad90c6438 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 8 Aug 2022 10:49:44 +0900
Subject: [PATCH 0359/3199] save

---
 ledger-tool/src/main.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/ledger-tool/src/main.rs b/ledger-tool/src/main.rs
index f1eaea430fbe71..83f801dba6ad24 100644
--- a/ledger-tool/src/main.rs
+++ b/ledger-tool/src/main.rs
@@ -141,7 +141,7 @@ fn output_entry(
     slot: Slot,
     entry_index: usize,
     entry: Entry,
-    preloader: Preloader,
+    preloader: &Preloader,
     to_schedule_stage: &mut Vec<Box<(SanitizedTransaction, Vec<LockAttempt>)>>,
     skip_voting: bool,
 ) {
@@ -368,7 +368,7 @@ fn output_slot(
                 slot,
                 entry_index,
                 entry,
-                preloader,
+                &preloader,
                 &mut txes,
                 skip_voting,
             );

From 2c5cdf59a5ae05b5d620e0fe3a24af1e5207fe60 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 8 Aug 2022 10:54:21 +0900
Subject: [PATCH 0360/3199] save

---
 scheduler/src/lib.rs | 12 +++++++++++-
 1 file changed, 11 insertions(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index cc550bc02e6f79..7a1f80cd5d0f5d 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -275,7 +275,17 @@ pub struct Preloader {
 
 impl Preloader {
     pub fn load(&self, address: Pubkey) -> PageRc {
-        todo!()
+        match self.book.entry(*address) {
+            AddressMapEntry::Vacant(book_entry) => {
+                let page = PageRc(ByAddress(MyRcInner::new(Page::new(CurrentUsage::new()))));
+                let cloned = PageRc::clone(&page);
+                book_entry.insert(page);
+                cloned
+            }
+            AddressMapEntry::Occupied(mut book_entry) => {
+                let page = book_entry.get_mut();
+                PageRc::clone(&page)
+        }
     }
 }
 

From 807e40922c5dcd842ecd983a8cbac3c9c488ab2f Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 8 Aug 2022 10:54:46 +0900
Subject: [PATCH 0361/3199] save

---
 scheduler/src/lib.rs | 1 +
 1 file changed, 1 insertion(+)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 7a1f80cd5d0f5d..c46935cd9dcb13 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -285,6 +285,7 @@ impl Preloader {
             AddressMapEntry::Occupied(mut book_entry) => {
                 let page = book_entry.get_mut();
                 PageRc::clone(&page)
+            }
         }
     }
 }

From c1bc7a48a68e89c52044ec2dfde9f38d6a91b5a9 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 8 Aug 2022 10:56:23 +0900
Subject: [PATCH 0362/3199] save

---
 scheduler/src/lib.rs | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index c46935cd9dcb13..5a99907cfef9f1 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -128,6 +128,10 @@ impl CurrentUsage {
             RequestedUsage::Writable => CurrentUsage::Writable,
         }
     }
+
+    fn placeholder() -> Self {
+        CurrentUsage::Readonly(0) // basically unused
+    }
 }
 
 #[derive(Clone, Copy, Debug)]

From fa03fcf0cf1116c701a056c5a304bfabef8ef7f2 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 8 Aug 2022 10:56:38 +0900
Subject: [PATCH 0363/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 5a99907cfef9f1..d9b32d765c2d10 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -281,7 +281,7 @@ impl Preloader {
     pub fn load(&self, address: Pubkey) -> PageRc {
         match self.book.entry(*address) {
             AddressMapEntry::Vacant(book_entry) => {
-                let page = PageRc(ByAddress(MyRcInner::new(Page::new(CurrentUsage::new()))));
+                let page = PageRc(ByAddress(MyRcInner::new(Page::new(CurrentUsage::placeholder()))));
                 let cloned = PageRc::clone(&page);
                 book_entry.insert(page);
                 cloned

From e5afa7d9c256568082f33fe75b76a03e95744c22 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 8 Aug 2022 10:56:58 +0900
Subject: [PATCH 0364/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index d9b32d765c2d10..3e8318ac1ff61e 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -274,7 +274,7 @@ impl AddressBook {
 }
 
 pub struct Preloader {
-    map: AddressMap,
+    book: AddressMap,
 }
 
 impl Preloader {

From eaa86a33108df3efa2531bbd552c05d30627f4d0 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 8 Aug 2022 10:57:06 +0900
Subject: [PATCH 0365/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 3e8318ac1ff61e..5adaec32eec7b5 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -269,7 +269,7 @@ impl AddressBook {
     }
 
     pub fn preloader(&self) -> Preloader {
-        Preloader{map: std::sync::Arc::clone(&self.book)}
+        Preloader{book: std::sync::Arc::clone(&self.book)}
     }
 }
 

From 4b99c764ea3dd5d237b75b56b67a3b068b95b1de Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 8 Aug 2022 10:57:14 +0900
Subject: [PATCH 0366/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 5adaec32eec7b5..11c9631f4529c5 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -279,7 +279,7 @@ pub struct Preloader {
 
 impl Preloader {
     pub fn load(&self, address: Pubkey) -> PageRc {
-        match self.book.entry(*address) {
+        match self.book.entry(address) {
             AddressMapEntry::Vacant(book_entry) => {
                 let page = PageRc(ByAddress(MyRcInner::new(Page::new(CurrentUsage::placeholder()))));
                 let cloned = PageRc::clone(&page);

From 4b89bcde369e91fafd5e6eff4dcedb548b0f3f1e Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 8 Aug 2022 11:00:21 +0900
Subject: [PATCH 0367/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 11c9631f4529c5..d52bb7764b4f12 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -287,7 +287,7 @@ impl Preloader {
                 cloned
             }
             AddressMapEntry::Occupied(mut book_entry) => {
-                let page = book_entry.get_mut();
+                let page = book_entry.get();
                 PageRc::clone(&page)
             }
         }

From a656c198289c398f346ce36e0ca2cb5f7b67d11f Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 8 Aug 2022 11:05:43 +0900
Subject: [PATCH 0368/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index d52bb7764b4f12..29fb4d2a1c31c2 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -536,7 +536,7 @@ impl ScheduleStage {
             );
 
             if !is_success {
-                //trace!("ensure_unlock_for_failed_execution(): {:?} {}", (&unique_weight, from_runnable), next_task.tx.signature());
+                trace!("ensure_unlock_for_failed_execution(): {:?} {}", (&unique_weight, from_runnable), next_task.tx.signature());
                 Self::ensure_unlock_for_failed_execution(
                     address_book,
                     &mut populated_lock_attempts,

From 65e7375fd0ddac1f94bb65236059d83820aa61a3 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 8 Aug 2022 11:05:52 +0900
Subject: [PATCH 0369/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 29fb4d2a1c31c2..716d69c26af59c 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -536,7 +536,7 @@ impl ScheduleStage {
             );
 
             if !is_success {
-                trace!("ensure_unlock_for_failed_execution(): {:?} {}", (&unique_weight, from_runnable), next_task.tx.signature());
+                trace!("ensure_unlock_for_failed_execution(): {:?} {}", (&unique_weight, from_runnable), next_task.tx.0.signature());
                 Self::ensure_unlock_for_failed_execution(
                     address_book,
                     &mut populated_lock_attempts,

From f765822311e00b493bb2baed510bb5bef3f87832 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 8 Aug 2022 11:08:19 +0900
Subject: [PATCH 0370/3199] save

---
 scheduler/src/lib.rs | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 716d69c26af59c..2ae8475824d7fe 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -129,8 +129,8 @@ impl CurrentUsage {
         }
     }
 
-    fn placeholder() -> Self {
-        CurrentUsage::Readonly(0) // basically unused
+    fn unused() -> Self {
+        CurrentUsage::Unused
     }
 }
 
@@ -281,7 +281,7 @@ impl Preloader {
     pub fn load(&self, address: Pubkey) -> PageRc {
         match self.book.entry(address) {
             AddressMapEntry::Vacant(book_entry) => {
-                let page = PageRc(ByAddress(MyRcInner::new(Page::new(CurrentUsage::placeholder()))));
+                let page = PageRc(ByAddress(MyRcInner::new(Page::new(CurrentUsage::unused()))));
                 let cloned = PageRc::clone(&page);
                 book_entry.insert(page);
                 cloned
@@ -536,7 +536,7 @@ impl ScheduleStage {
             );
 
             if !is_success {
-                trace!("ensure_unlock_for_failed_execution(): {:?} {}", (&unique_weight, from_runnable), next_task.tx.0.signature());
+                //trace!("ensure_unlock_for_failed_execution(): {:?} {}", (&unique_weight, from_runnable), next_task.tx.0.signature());
                 Self::ensure_unlock_for_failed_execution(
                     address_book,
                     &mut populated_lock_attempts,

From 64bc2620650327c4df939549d79213805533dce7 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 8 Aug 2022 11:19:32 +0900
Subject: [PATCH 0371/3199] save

---
 transaction-scheduler-bench/src/main.rs | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/transaction-scheduler-bench/src/main.rs b/transaction-scheduler-bench/src/main.rs
index 3de261c828692b..cd3ff80d05d296 100644
--- a/transaction-scheduler-bench/src/main.rs
+++ b/transaction-scheduler-bench/src/main.rs
@@ -117,6 +117,7 @@ struct PacketSendingConfig {
 }
 
 fn spawn_unified_scheduler(
+        address_book: solana_scheduler::AddressBook,
         num_execution_threads: usize,
         packet_batch_receiver: Receiver<BatchSenderMessage>,
         transaction_batch_senders: Vec<Sender<TransactionBatchMessage>>,
@@ -128,7 +129,6 @@ fn spawn_unified_scheduler(
     std::thread::Builder::new().name("sol-scheduler".to_string()).spawn(move || {
         let mut runnable_queue = solana_scheduler::TaskQueue::default();
         let mut contended_queue = solana_scheduler::TaskQueue::default();
-        let mut address_book = solana_scheduler::AddressBook::default();
 
         solana_scheduler::ScheduleStage::run(
             num_execution_threads * 10,
@@ -144,6 +144,8 @@ fn spawn_unified_scheduler(
 
 fn main() {
     solana_logger::setup_with_default("INFO");
+    let mut address_book = solana_scheduler::AddressBook::default();
+    let preloader = address_book.preloader();
 
     let Args {
         packet_send_rate,
@@ -169,6 +171,7 @@ fn main() {
 
     // Spawns and runs the scheduler thread
     let scheduler_handle = spawn_unified_scheduler(
+        address_book,
         num_execution_threads,
         packet_batch_receiver,
         transaction_batch_senders,

From 654f122b67ad1f3d6ca0c79fb7d0f67b43073593 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 8 Aug 2022 11:20:01 +0900
Subject: [PATCH 0372/3199] save

---
 transaction-scheduler-bench/src/main.rs | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/transaction-scheduler-bench/src/main.rs b/transaction-scheduler-bench/src/main.rs
index cd3ff80d05d296..318e1b1eda3a3e 100644
--- a/transaction-scheduler-bench/src/main.rs
+++ b/transaction-scheduler-bench/src/main.rs
@@ -205,6 +205,7 @@ fn main() {
         num_write_locks_per_tx: num_read_write_locks_per_tx,
     });
     let packet_sender_handles = spawn_packet_senders(
+        preloader,
         metrics.clone(),
         high_conflict_sender,
         accounts,
@@ -347,6 +348,7 @@ fn handle_transaction_batch(
 const NUM_SENDERS: usize = 1;
 
 fn spawn_packet_senders(
+    preloader: &solana_scheduler::Preloader,
     metrics: Arc<TransactionSchedulerBenchMetrics>,
     high_conflict_sender: usize,
     accounts: Arc<Vec<Keypair>>,

From 8f7fdfa71e0a0b95b631044862fbe770bd6155f8 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 8 Aug 2022 11:20:09 +0900
Subject: [PATCH 0373/3199] save

---
 transaction-scheduler-bench/src/main.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/transaction-scheduler-bench/src/main.rs b/transaction-scheduler-bench/src/main.rs
index 318e1b1eda3a3e..07fd0c43d590d5 100644
--- a/transaction-scheduler-bench/src/main.rs
+++ b/transaction-scheduler-bench/src/main.rs
@@ -205,7 +205,7 @@ fn main() {
         num_write_locks_per_tx: num_read_write_locks_per_tx,
     });
     let packet_sender_handles = spawn_packet_senders(
-        preloader,
+        &preloader,
         metrics.clone(),
         high_conflict_sender,
         accounts,

From df00a124a8f52efcb2d11f1f3fd38711c6ff2f28 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 8 Aug 2022 11:20:38 +0900
Subject: [PATCH 0374/3199] save

---
 transaction-scheduler-bench/src/main.rs | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/transaction-scheduler-bench/src/main.rs b/transaction-scheduler-bench/src/main.rs
index 07fd0c43d590d5..2838a28d7f08d4 100644
--- a/transaction-scheduler-bench/src/main.rs
+++ b/transaction-scheduler-bench/src/main.rs
@@ -365,6 +365,7 @@ fn spawn_packet_senders(
                 accounts.len()
             };
             spawn_packet_sender(
+                preloader,
                 metrics.clone(),
                 num_accounts,
                 accounts.clone(),
@@ -378,6 +379,7 @@ fn spawn_packet_senders(
 }
 
 fn spawn_packet_sender(
+    preloader: &solana_scheduler::Preloader,
     metrics: Arc<TransactionSchedulerBenchMetrics>,
     num_accounts: usize,
     accounts: Arc<Vec<Keypair>>,

From 2c7a67f1d918a6770ae390e84141562dae6238b4 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 8 Aug 2022 11:21:14 +0900
Subject: [PATCH 0375/3199] save

---
 transaction-scheduler-bench/src/main.rs | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/transaction-scheduler-bench/src/main.rs b/transaction-scheduler-bench/src/main.rs
index 2838a28d7f08d4..3621e81a178772 100644
--- a/transaction-scheduler-bench/src/main.rs
+++ b/transaction-scheduler-bench/src/main.rs
@@ -402,6 +402,7 @@ fn spawn_packet_sender(
 }
 
 fn send_packets(
+    preloader: &solana_scheduler::Preloader,
     metrics: Arc<TransactionSchedulerBenchMetrics>,
     num_accounts: usize,
     accounts: Arc<Vec<Keypair>>,
@@ -429,6 +430,7 @@ fn send_packets(
             break;
         }
         let (packet_batches, packet_build_time) = measure!(build_packet_batches(
+            preloader: &solana_scheduler::Preloader,
             &config,
             num_accounts,
             &accounts,
@@ -445,6 +447,7 @@ fn send_packets(
 }
 
 fn build_packet_batches(
+    preloader: &solana_scheduler::Preloader,
     config: &PacketSendingConfig,
     num_accounts: usize,
     accounts: &[Keypair],
@@ -462,11 +465,12 @@ fn build_packet_batch(
     blockhash: &Hash,
 ) -> Vec<PreprocessedTransaction> {
     (0..config.packets_per_batch)
-        .map(|_| build_packet(config, num_accounts, accounts, blockhash))
+        .map(|_| build_packet(preloader, config, num_accounts, accounts, blockhash))
         .collect()
 }
 
 fn build_packet(
+    preloader: &solana_scheduler::Preloader,
     config: &PacketSendingConfig,
     num_accounts: usize,
     accounts: &[Keypair],

From 2818b97316389899e9d2b5c52a8e88015d79b9f7 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 8 Aug 2022 11:21:32 +0900
Subject: [PATCH 0376/3199] save

---
 transaction-scheduler-bench/src/main.rs | 1 +
 1 file changed, 1 insertion(+)

diff --git a/transaction-scheduler-bench/src/main.rs b/transaction-scheduler-bench/src/main.rs
index 3621e81a178772..51f111433a12f9 100644
--- a/transaction-scheduler-bench/src/main.rs
+++ b/transaction-scheduler-bench/src/main.rs
@@ -390,6 +390,7 @@ fn spawn_packet_sender(
 ) -> JoinHandle<()> {
     std::thread::Builder::new().name("sol-producer".to_string()).spawn(move || {
         send_packets(
+            preloader,
             metrics,
             num_accounts,
             accounts,

From 47aa3f338ca7ad8405c691e190e5ec0ecfd7ee6d Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 8 Aug 2022 11:21:48 +0900
Subject: [PATCH 0377/3199] save

---
 transaction-scheduler-bench/src/main.rs | 1 +
 1 file changed, 1 insertion(+)

diff --git a/transaction-scheduler-bench/src/main.rs b/transaction-scheduler-bench/src/main.rs
index 51f111433a12f9..a90b20bd7f4484 100644
--- a/transaction-scheduler-bench/src/main.rs
+++ b/transaction-scheduler-bench/src/main.rs
@@ -460,6 +460,7 @@ fn build_packet_batches(
 }
 
 fn build_packet_batch(
+    preloader: &solana_scheduler::Preloader,
     config: &PacketSendingConfig,
     num_accounts: usize,
     accounts: &[Keypair],

From f459c48e669af4b047178a0e0a5fce42cb62bc69 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 8 Aug 2022 11:22:00 +0900
Subject: [PATCH 0378/3199] save

---
 transaction-scheduler-bench/src/main.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/transaction-scheduler-bench/src/main.rs b/transaction-scheduler-bench/src/main.rs
index a90b20bd7f4484..868297d4808df8 100644
--- a/transaction-scheduler-bench/src/main.rs
+++ b/transaction-scheduler-bench/src/main.rs
@@ -455,7 +455,7 @@ fn build_packet_batches(
     blockhash: &Hash,
 ) -> Vec<Vec<PreprocessedTransaction>> {
     (0..config.batches_per_msg)
-        .map(|_| build_packet_batch(config, num_accounts, accounts, blockhash))
+        .map(|_| build_packet_batch(preloader, config, num_accounts, accounts, blockhash))
         .collect()
 }
 

From 3cc0d57866fef6521077addfdc9e48cee8986670 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 8 Aug 2022 11:22:49 +0900
Subject: [PATCH 0379/3199] save

---
 transaction-scheduler-bench/src/main.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/transaction-scheduler-bench/src/main.rs b/transaction-scheduler-bench/src/main.rs
index 868297d4808df8..93074fb7fad5be 100644
--- a/transaction-scheduler-bench/src/main.rs
+++ b/transaction-scheduler-bench/src/main.rs
@@ -507,11 +507,11 @@ fn build_packet(
     let writable_lock_iter = locks
         .writable
         .iter()
-        .map(|address| solana_scheduler::LockAttempt::new(**address, solana_scheduler::RequestedUsage::Writable));
+        .map(|address| solana_scheduler::LockAttempt::new(preloader.load(**address), solana_scheduler::RequestedUsage::Writable));
     let readonly_lock_iter = locks
         .readonly
         .iter()
-        .map(|address| solana_scheduler::LockAttempt::new(**address, solana_scheduler::RequestedUsage::Readonly));
+        .map(|address| solana_scheduler::LockAttempt::new(preloader.load(**address), solana_scheduler::RequestedUsage::Readonly));
     let locks = writable_lock_iter.chain(readonly_lock_iter).collect::<Vec<_>>();
 
     Box::new((sanitized_tx, locks))

From defd2a8355ac09c3458e3e019a759cc54b6e42c2 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 8 Aug 2022 11:23:18 +0900
Subject: [PATCH 0380/3199] save

---
 transaction-scheduler-bench/src/main.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/transaction-scheduler-bench/src/main.rs b/transaction-scheduler-bench/src/main.rs
index 93074fb7fad5be..73553b127a89eb 100644
--- a/transaction-scheduler-bench/src/main.rs
+++ b/transaction-scheduler-bench/src/main.rs
@@ -431,7 +431,7 @@ fn send_packets(
             break;
         }
         let (packet_batches, packet_build_time) = measure!(build_packet_batches(
-            preloader: &solana_scheduler::Preloader,
+            preloader,
             &config,
             num_accounts,
             &accounts,

From 66c59cd60587b04121bf4a887dbffa62c5503b06 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 8 Aug 2022 11:24:32 +0900
Subject: [PATCH 0381/3199] save

---
 transaction-scheduler-bench/src/main.rs | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/transaction-scheduler-bench/src/main.rs b/transaction-scheduler-bench/src/main.rs
index 73553b127a89eb..4543f341f6c800 100644
--- a/transaction-scheduler-bench/src/main.rs
+++ b/transaction-scheduler-bench/src/main.rs
@@ -348,7 +348,7 @@ fn handle_transaction_batch(
 const NUM_SENDERS: usize = 1;
 
 fn spawn_packet_senders(
-    preloader: &solana_scheduler::Preloader,
+    preloader: Arc<solana_scheduler::Preloader>,
     metrics: Arc<TransactionSchedulerBenchMetrics>,
     high_conflict_sender: usize,
     accounts: Arc<Vec<Keypair>>,
@@ -379,7 +379,7 @@ fn spawn_packet_senders(
 }
 
 fn spawn_packet_sender(
-    preloader: &solana_scheduler::Preloader,
+    preloader: Arc<solana_scheduler::Preloader>,
     metrics: Arc<TransactionSchedulerBenchMetrics>,
     num_accounts: usize,
     accounts: Arc<Vec<Keypair>>,
@@ -403,7 +403,7 @@ fn spawn_packet_sender(
 }
 
 fn send_packets(
-    preloader: &solana_scheduler::Preloader,
+    preloader: Arc<solana_scheduler::Preloader>,
     metrics: Arc<TransactionSchedulerBenchMetrics>,
     num_accounts: usize,
     accounts: Arc<Vec<Keypair>>,
@@ -448,7 +448,7 @@ fn send_packets(
 }
 
 fn build_packet_batches(
-    preloader: &solana_scheduler::Preloader,
+    preloader: Arc<solana_scheduler::Preloader>,
     config: &PacketSendingConfig,
     num_accounts: usize,
     accounts: &[Keypair],
@@ -460,7 +460,7 @@ fn build_packet_batches(
 }
 
 fn build_packet_batch(
-    preloader: &solana_scheduler::Preloader,
+    preloader: Arc<solana_scheduler::Preloader>,
     config: &PacketSendingConfig,
     num_accounts: usize,
     accounts: &[Keypair],
@@ -472,7 +472,7 @@ fn build_packet_batch(
 }
 
 fn build_packet(
-    preloader: &solana_scheduler::Preloader,
+    preloader: Arc<solana_scheduler::Preloader>,
     config: &PacketSendingConfig,
     num_accounts: usize,
     accounts: &[Keypair],

From 99c4288ae62f52edf28ab560897ad1d29db0add0 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 8 Aug 2022 11:24:48 +0900
Subject: [PATCH 0382/3199] save

---
 transaction-scheduler-bench/src/main.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/transaction-scheduler-bench/src/main.rs b/transaction-scheduler-bench/src/main.rs
index 4543f341f6c800..a8ff5011438024 100644
--- a/transaction-scheduler-bench/src/main.rs
+++ b/transaction-scheduler-bench/src/main.rs
@@ -205,7 +205,7 @@ fn main() {
         num_write_locks_per_tx: num_read_write_locks_per_tx,
     });
     let packet_sender_handles = spawn_packet_senders(
-        &preloader,
+        Arc::new(preloader),
         metrics.clone(),
         high_conflict_sender,
         accounts,

From 5cb78c4dd54ad9ece80c0e9f4b43b81be9c7ec45 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 8 Aug 2022 11:25:32 +0900
Subject: [PATCH 0383/3199] save

---
 transaction-scheduler-bench/src/main.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/transaction-scheduler-bench/src/main.rs b/transaction-scheduler-bench/src/main.rs
index a8ff5011438024..bf8d96035582ee 100644
--- a/transaction-scheduler-bench/src/main.rs
+++ b/transaction-scheduler-bench/src/main.rs
@@ -117,7 +117,7 @@ struct PacketSendingConfig {
 }
 
 fn spawn_unified_scheduler(
-        address_book: solana_scheduler::AddressBook,
+        mut address_book: solana_scheduler::AddressBook,
         num_execution_threads: usize,
         packet_batch_receiver: Receiver<BatchSenderMessage>,
         transaction_batch_senders: Vec<Sender<TransactionBatchMessage>>,
@@ -365,7 +365,7 @@ fn spawn_packet_senders(
                 accounts.len()
             };
             spawn_packet_sender(
-                preloader,
+                Arc::clone(preloader),
                 metrics.clone(),
                 num_accounts,
                 accounts.clone(),

From d2a7917b1b97e97bbf57913754262b130a92f820 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 8 Aug 2022 11:25:48 +0900
Subject: [PATCH 0384/3199] save

---
 transaction-scheduler-bench/src/main.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/transaction-scheduler-bench/src/main.rs b/transaction-scheduler-bench/src/main.rs
index bf8d96035582ee..15a803db686acb 100644
--- a/transaction-scheduler-bench/src/main.rs
+++ b/transaction-scheduler-bench/src/main.rs
@@ -365,7 +365,7 @@ fn spawn_packet_senders(
                 accounts.len()
             };
             spawn_packet_sender(
-                Arc::clone(preloader),
+                Arc::clone(&preloader),
                 metrics.clone(),
                 num_accounts,
                 accounts.clone(),

From 91f3ea2f1c64d769f8d3850752d9e99823bb0bad Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 8 Aug 2022 11:26:42 +0900
Subject: [PATCH 0385/3199] save

---
 transaction-scheduler-bench/src/main.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/transaction-scheduler-bench/src/main.rs b/transaction-scheduler-bench/src/main.rs
index 15a803db686acb..030f10aece1ffd 100644
--- a/transaction-scheduler-bench/src/main.rs
+++ b/transaction-scheduler-bench/src/main.rs
@@ -431,7 +431,7 @@ fn send_packets(
             break;
         }
         let (packet_batches, packet_build_time) = measure!(build_packet_batches(
-            preloader,
+            &preloader,
             &config,
             num_accounts,
             &accounts,
@@ -448,7 +448,7 @@ fn send_packets(
 }
 
 fn build_packet_batches(
-    preloader: Arc<solana_scheduler::Preloader>,
+    preloader: &solana_scheduler::Preloader,
     config: &PacketSendingConfig,
     num_accounts: usize,
     accounts: &[Keypair],

From c64746383ee0f42828feef37c8235e28abc03098 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 8 Aug 2022 11:27:03 +0900
Subject: [PATCH 0386/3199] save

---
 transaction-scheduler-bench/src/main.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/transaction-scheduler-bench/src/main.rs b/transaction-scheduler-bench/src/main.rs
index 030f10aece1ffd..9aee9dd2b5ecea 100644
--- a/transaction-scheduler-bench/src/main.rs
+++ b/transaction-scheduler-bench/src/main.rs
@@ -460,7 +460,7 @@ fn build_packet_batches(
 }
 
 fn build_packet_batch(
-    preloader: Arc<solana_scheduler::Preloader>,
+    preloader: &solana_scheduler::Preloader,
     config: &PacketSendingConfig,
     num_accounts: usize,
     accounts: &[Keypair],
@@ -472,7 +472,7 @@ fn build_packet_batch(
 }
 
 fn build_packet(
-    preloader: Arc<solana_scheduler::Preloader>,
+    preloader: &solana_scheduler::Preloader,
     config: &PacketSendingConfig,
     num_accounts: usize,
     accounts: &[Keypair],

From 3e151c58b091e713f3fbbf964a4fa3c7e5ecaf51 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 8 Aug 2022 11:35:42 +0900
Subject: [PATCH 0387/3199] save

---
 ledger-tool/src/main.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ledger-tool/src/main.rs b/ledger-tool/src/main.rs
index 83f801dba6ad24..15463aa9a65988 100644
--- a/ledger-tool/src/main.rs
+++ b/ledger-tool/src/main.rs
@@ -375,7 +375,7 @@ fn output_slot(
         }
 
         let mut weight = 10_000_000;
-        for i in 0..10000 {
+        for i in 0..100 {
             error!("started!: {} {}", i, txes.len());
             for tx in txes.clone() {
                 while depth.load(Ordering::Relaxed) > 10_000 {

From 34fbbe80ebdc1df65f21645e5d32160670fd1241 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 8 Aug 2022 11:38:17 +0900
Subject: [PATCH 0388/3199] save

---
 transaction-scheduler-bench/src/main.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/transaction-scheduler-bench/src/main.rs b/transaction-scheduler-bench/src/main.rs
index 9aee9dd2b5ecea..fc38fab0d3be19 100644
--- a/transaction-scheduler-bench/src/main.rs
+++ b/transaction-scheduler-bench/src/main.rs
@@ -131,7 +131,7 @@ fn spawn_unified_scheduler(
         let mut contended_queue = solana_scheduler::TaskQueue::default();
 
         solana_scheduler::ScheduleStage::run(
-            num_execution_threads * 10,
+            num_execution_threads * 100,
             &mut runnable_queue,
             &mut contended_queue,
             &mut address_book,

From 8f20fad174af8034ae4ab01bd9f5ea79f90ce72d Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 8 Aug 2022 11:42:15 +0900
Subject: [PATCH 0389/3199] save

---
 transaction-scheduler-bench/src/main.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/transaction-scheduler-bench/src/main.rs b/transaction-scheduler-bench/src/main.rs
index fc38fab0d3be19..9aee9dd2b5ecea 100644
--- a/transaction-scheduler-bench/src/main.rs
+++ b/transaction-scheduler-bench/src/main.rs
@@ -131,7 +131,7 @@ fn spawn_unified_scheduler(
         let mut contended_queue = solana_scheduler::TaskQueue::default();
 
         solana_scheduler::ScheduleStage::run(
-            num_execution_threads * 100,
+            num_execution_threads * 10,
             &mut runnable_queue,
             &mut contended_queue,
             &mut address_book,

From d5fc0ce86a1874750e4b81935d0ae20fcc385ba8 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 9 Aug 2022 09:23:54 +0900
Subject: [PATCH 0390/3199] save

---
 transaction-scheduler-bench/src/main.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/transaction-scheduler-bench/src/main.rs b/transaction-scheduler-bench/src/main.rs
index 9aee9dd2b5ecea..dd32b6517e502e 100644
--- a/transaction-scheduler-bench/src/main.rs
+++ b/transaction-scheduler-bench/src/main.rs
@@ -503,7 +503,7 @@ fn build_packet(
     )
     .unwrap();
 
-    let locks = sanitized_tx.get_account_locks().unwrap();
+    let locks = sanitized_tx.get_account_locks_unchecked().unwrap();
     let writable_lock_iter = locks
         .writable
         .iter()

From 10338edc1f14b7a43afaf5c685b12a0fb1e588b2 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 9 Aug 2022 09:24:14 +0900
Subject: [PATCH 0391/3199] save

---
 transaction-scheduler-bench/src/main.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/transaction-scheduler-bench/src/main.rs b/transaction-scheduler-bench/src/main.rs
index dd32b6517e502e..558977de7b083a 100644
--- a/transaction-scheduler-bench/src/main.rs
+++ b/transaction-scheduler-bench/src/main.rs
@@ -503,7 +503,7 @@ fn build_packet(
     )
     .unwrap();
 
-    let locks = sanitized_tx.get_account_locks_unchecked().unwrap();
+    let locks = sanitized_tx.get_account_locks_unchecked();
     let writable_lock_iter = locks
         .writable
         .iter()

From 513cf93bf94059bcec2dec98f37e85db2df61c4b Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 9 Aug 2022 10:15:20 +0900
Subject: [PATCH 0392/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 2ae8475824d7fe..36c6350a53ea1b 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -453,7 +453,7 @@ impl ScheduleStage {
     #[inline(never)]
     fn get_weight_from_contended(address_book: &AddressBook) -> Option<UniqueWeight> {
         let mut heaviest_weight: Option<UniqueWeight> = None;
-        //error!("n u a len(): {}", address_book.newly_uncontended_addresses.len());
+        trace!("n_u_a len(): {}", address_book.newly_uncontended_addresses.len());
         for page in address_book.newly_uncontended_addresses.iter() {
             let newly_uncontended_unique_weights = &page.0.contended_unique_weights;
             if let Some(&weight) = newly_uncontended_unique_weights.last() {

From 38686b3c0dfcccb61f2cf8bd0b16392a012c54a1 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 9 Aug 2022 10:27:13 +0900
Subject: [PATCH 0393/3199] save

---
 scheduler/src/lib.rs | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 36c6350a53ea1b..f3e5b1b5ea2596 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -718,7 +718,12 @@ impl ScheduleStage {
                 }
             }
 
-            while executing_queue_count < max_executing_queue_count {
+            loop { 
+                if executing_queue_count < max_executing_queue_count {
+                    trace!("outgoing queue full");
+                    break;
+                }
+
                 let maybe_ee =
                     Self::schedule_next_execution(runnable_queue, contended_queue, address_book);
                 if let Some(ee) = maybe_ee {
@@ -727,6 +732,7 @@ impl ScheduleStage {
 
                     to_execute_substage.send(ee).unwrap();
                 } else {
+                    trace!("incoming queue starved");
                     break;
                 }
             }

From b9b7f8c97670bf93fa2053389cb239c31a6884ef Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 9 Aug 2022 10:27:19 +0900
Subject: [PATCH 0394/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index f3e5b1b5ea2596..d33435d7a3f61e 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -718,7 +718,7 @@ impl ScheduleStage {
                 }
             }
 
-            loop { 
+            loop {
                 if executing_queue_count < max_executing_queue_count {
                     trace!("outgoing queue full");
                     break;

From 370d609d7fad060ce87f68a619e6910f3b37fbdd Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 9 Aug 2022 10:30:27 +0900
Subject: [PATCH 0395/3199] save

---
 scheduler/src/lib.rs | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index d33435d7a3f61e..3f0a98a81b6e45 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -483,18 +483,24 @@ impl ScheduleStage {
             Self::get_weight_from_contended(address_book),
         ) {
             (Some(heaviest_runnable_entry), None) => {
+                trace!("runnable only");
                 Some((Some(contended_queue), heaviest_runnable_entry))
             }
-            (None, Some(weight_from_contended)) => Some((
+            (None, Some(weight_from_contended)) => {
+                trace!("contended only");
+                Some((
                 None,
                 contended_queue.entry_to_execute(weight_from_contended),
-            )),
+            ))
+            },
             (Some(heaviest_runnable_entry), Some(weight_from_contended)) => {
                 let weight_from_runnable = heaviest_runnable_entry.key();
 
                 if weight_from_runnable > &weight_from_contended {
+                    trace!("runnable > contended");
                     Some((Some(contended_queue), heaviest_runnable_entry))
                 } else if &weight_from_contended > weight_from_runnable {
+                    trace!("contended > runnnable");
                     Some((
                         None,
                         contended_queue.entry_to_execute(weight_from_contended),

From 59b02bc01912bf072f496786113fe28d73442386 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 9 Aug 2022 10:34:42 +0900
Subject: [PATCH 0396/3199] save

---
 scheduler/src/lib.rs | 14 ++++++++------
 1 file changed, 8 insertions(+), 6 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 3f0a98a81b6e45..6ef4f9cb857c0f 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -483,11 +483,11 @@ impl ScheduleStage {
             Self::get_weight_from_contended(address_book),
         ) {
             (Some(heaviest_runnable_entry), None) => {
-                trace!("runnable only");
+                trace!("select: runnable only");
                 Some((Some(contended_queue), heaviest_runnable_entry))
             }
             (None, Some(weight_from_contended)) => {
-                trace!("contended only");
+                trace!("select: contended only");
                 Some((
                 None,
                 contended_queue.entry_to_execute(weight_from_contended),
@@ -497,10 +497,10 @@ impl ScheduleStage {
                 let weight_from_runnable = heaviest_runnable_entry.key();
 
                 if weight_from_runnable > &weight_from_contended {
-                    trace!("runnable > contended");
+                    trace!("select: runnable > contended");
                     Some((Some(contended_queue), heaviest_runnable_entry))
                 } else if &weight_from_contended > weight_from_runnable {
-                    trace!("contended > runnnable");
+                    trace!("select: contended > runnnable");
                     Some((
                         None,
                         contended_queue.entry_to_execute(weight_from_contended),
@@ -703,11 +703,13 @@ impl ScheduleStage {
                                 let maybe_ee =
                                     Self::schedule_next_execution(runnable_queue, contended_queue, address_book);
                                 if let Some(ee) = maybe_ee {
-                                    trace!("send to execute");
+                                    trace!("batched: send to execute");
                                     executing_queue_count += 1;
 
                                     to_execute_substage.send(ee).unwrap();
                                 }
+                            } else {
+                                    trace!("batched: outgoing queue full");
                             }
                         }
                     }
@@ -725,7 +727,7 @@ impl ScheduleStage {
             }
 
             loop {
-                if executing_queue_count < max_executing_queue_count {
+                if executing_queue_count >= max_executing_queue_count {
                     trace!("outgoing queue full");
                     break;
                 }

From 31db674c318f933216db11f5e7ce055253e94067 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 9 Aug 2022 10:46:17 +0900
Subject: [PATCH 0397/3199] save

---
 scheduler/src/lib.rs | 14 ++++++++------
 1 file changed, 8 insertions(+), 6 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 6ef4f9cb857c0f..c93f8c301ffd44 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -386,7 +386,7 @@ type PreprocessedTransaction = (SanitizedTransaction, Vec<LockAttempt>);
 // multiplexed to reduce the futex syscal per tx down to minimum and to make the schduler to
 // adaptive relative load between sigverify stage and execution substage
 // switched from crossbeam_channel::select! due to observed poor performance
-pub enum MultiplexedPayload {
+pub enum Multiplexed {
     FromPrevious((Weight, Box<PreprocessedTransaction>)),
     FromPreviousBatched(Vec<Vec<Box<PreprocessedTransaction>>>),
     FromExecute(Box<ExecutionEnvironment>),
@@ -675,7 +675,7 @@ impl ScheduleStage {
         runnable_queue: &mut TaskQueue,
         contended_queue: &mut TaskQueue,
         address_book: &mut AddressBook,
-        from: &crossbeam_channel::Receiver<MultiplexedPayload>,
+        from: &crossbeam_channel::Receiver<Multiplexed>,
         to_execute_substage: &crossbeam_channel::Sender<Box<ExecutionEnvironment>>,
         to_next_stage: Option<&crossbeam_channel::Sender<Box<ExecutionEnvironment>>>, // assume nonblocking
     ) {
@@ -687,12 +687,12 @@ impl ScheduleStage {
 
             let i = from.recv().unwrap();
             match i {
-                MultiplexedPayload::FromPrevious(weighted_tx) => {
+                Multiplexed::FromPrevious(weighted_tx) => {
                     trace!("recv from previous");
 
                     Self::register_runnable_task(weighted_tx, runnable_queue, &mut current_unique_key);
                 }
-                MultiplexedPayload::FromPreviousBatched(vvv) => {
+                Multiplexed::FromPreviousBatched(vvv) => {
                     trace!("recv from previous");
 
                     for vv in vvv {
@@ -714,7 +714,7 @@ impl ScheduleStage {
                         }
                     }
                 }
-                MultiplexedPayload::FromExecute(mut processed_execution_environment) => {
+                Multiplexed::FromExecute(mut processed_execution_environment) => {
                     trace!("recv from execute");
                     executing_queue_count -= 1;
 
@@ -727,7 +727,9 @@ impl ScheduleStage {
             }
 
             loop {
-                if executing_queue_count >= max_executing_queue_count {
+                if !address_book.newly_uncontended_addresses.is_empty() {
+                    trace!("prefer emptying n_u_a");
+                } else if executing_queue_count >= max_executing_queue_count {
                     trace!("outgoing queue full");
                     break;
                 }

From 50a06618f7c0ca74954edd1da8a35069030d0ace Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 9 Aug 2022 10:46:52 +0900
Subject: [PATCH 0398/3199] save

---
 transaction-scheduler-bench/src/main.rs | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/transaction-scheduler-bench/src/main.rs b/transaction-scheduler-bench/src/main.rs
index 558977de7b083a..8db94f59350594 100644
--- a/transaction-scheduler-bench/src/main.rs
+++ b/transaction-scheduler-bench/src/main.rs
@@ -80,9 +80,9 @@ struct Args {
 /// Some convenient type aliases
 type PreprocessedTransaction = Box<(SanitizedTransaction, Vec<solana_scheduler::LockAttempt>)>;
 type TransactionMessage = PreprocessedTransaction;
-type CompletedTransactionMessage = solana_scheduler::MultiplexedPayload; // (usize, Box<solana_scheduler::ExecutionEnvironment>); //(usize, TransactionMessage); // thread index and transaction message
+type CompletedTransactionMessage = solana_scheduler::Multiplexed; // (usize, Box<solana_scheduler::ExecutionEnvironment>); //(usize, TransactionMessage); // thread index and transaction message
 type TransactionBatchMessage = Box<solana_scheduler::ExecutionEnvironment>; // Vec<TransactionMessage>;
-type BatchSenderMessage = solana_scheduler::MultiplexedPayload; // Vec<Vec<PreprocessedTransaction>>;
+type BatchSenderMessage = solana_scheduler::Multiplexed; // Vec<Vec<PreprocessedTransaction>>;
 
 #[derive(Debug, Default)]
 struct TransactionSchedulerBenchMetrics {
@@ -341,7 +341,7 @@ fn handle_transaction_batch(
         .fetch_add(priority_collected, Ordering::Relaxed);
 
         completed_transaction_sender
-            .send(solana_scheduler::MultiplexedPayload::FromExecute(transaction_batch))
+            .send(solana_scheduler::Multiplexed::FromExecute(transaction_batch))
             .unwrap();
 }
 
@@ -441,7 +441,7 @@ fn send_packets(
             packet_batches.iter().map(|pb| pb.len()).sum(),
             Ordering::Relaxed,
         );
-        let _ = packet_batch_sender.send(solana_scheduler::MultiplexedPayload::FromPreviousBatched(packet_batches));
+        let _ = packet_batch_sender.send(solana_scheduler::Multiplexed::FromPreviousBatched(packet_batches));
 
         std::thread::sleep(loop_duration.saturating_sub(packet_build_time.as_duration()));
     }

From 3e830d2a0b2409af363583143193200ebf6e429f Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 9 Aug 2022 10:54:31 +0900
Subject: [PATCH 0399/3199] save

---
 scheduler/src/lib.rs | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index c93f8c301ffd44..e8927f721b5e12 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -543,6 +543,7 @@ impl ScheduleStage {
 
             if !is_success {
                 //trace!("ensure_unlock_for_failed_execution(): {:?} {}", (&unique_weight, from_runnable), next_task.tx.0.signature());
+                trace!("move to contended due to lock failure");
                 Self::ensure_unlock_for_failed_execution(
                     address_book,
                     &mut populated_lock_attempts,
@@ -558,6 +559,7 @@ impl ScheduleStage {
                 continue;
             }
 
+            trace!("successful lock");
             Self::finalize_successful_lock_before_execution(
                 address_book,
                 &unique_weight,
@@ -736,6 +738,7 @@ impl ScheduleStage {
 
                 let maybe_ee =
                     Self::schedule_next_execution(runnable_queue, contended_queue, address_book);
+
                 if let Some(ee) = maybe_ee {
                     trace!("send to execute");
                     executing_queue_count += 1;

From 3def36a6e71a03a7f1361e8443fe9ef798615846 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 9 Aug 2022 11:01:07 +0900
Subject: [PATCH 0400/3199] save

---
 scheduler/src/lib.rs | 14 ++++++--------
 1 file changed, 6 insertions(+), 8 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index e8927f721b5e12..47c0ea150ab5b7 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -543,18 +543,19 @@ impl ScheduleStage {
 
             if !is_success {
                 //trace!("ensure_unlock_for_failed_execution(): {:?} {}", (&unique_weight, from_runnable), next_task.tx.0.signature());
-                trace!("move to contended due to lock failure");
                 Self::ensure_unlock_for_failed_execution(
                     address_book,
                     &mut populated_lock_attempts,
-                    from_runnable,
                 );
                 std::mem::swap(&mut next_task.tx.1, &mut populated_lock_attempts);
                 if from_runnable {
+                    trace!("move to contended due to lock failure");
                     reborrowed_contended_queue
                         .unwrap()
                         .add_to_schedule(*queue_entry.key(), queue_entry.remove());
                     // maybe run lightweight prune logic on contended_queue here.
+                } else {
+                    trace!("relock failed; remains in contended");
                 }
                 continue;
             }
@@ -592,17 +593,14 @@ impl ScheduleStage {
     fn ensure_unlock_for_failed_execution(
         address_book: &mut AddressBook,
         lock_attempts: &mut Vec<LockAttempt>,
-        from_runnable: bool,
     ) {
         for l in lock_attempts {
             address_book.ensure_unlock(l);
 
             // revert because now contended again
-            if !from_runnable {
-                //error!("n u a len() before: {}", address_book.newly_uncontended_addresses.len());
-                address_book.newly_uncontended_addresses.remove(&l.target);
-                //error!("n u a len() after: {}", address_book.newly_uncontended_addresses.len());
-            }
+            //error!("n u a len() before: {}", address_book.newly_uncontended_addresses.len());
+            address_book.newly_uncontended_addresses.remove(&l.target);
+            //error!("n u a len() after: {}", address_book.newly_uncontended_addresses.len());
 
             // todo: mem::forget and panic in LockAttempt::drop()
         }

From 4856a925e4611d4ad922e89f55fa25873c7f4f82 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 9 Aug 2022 11:08:28 +0900
Subject: [PATCH 0401/3199] save

---
 scheduler/src/lib.rs | 10 +++++++---
 1 file changed, 7 insertions(+), 3 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 47c0ea150ab5b7..663396ae23b282 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -546,6 +546,7 @@ impl ScheduleStage {
                 Self::ensure_unlock_for_failed_execution(
                     address_book,
                     &mut populated_lock_attempts,
+                    from_runnable,
                 );
                 std::mem::swap(&mut next_task.tx.1, &mut populated_lock_attempts);
                 if from_runnable {
@@ -593,14 +594,17 @@ impl ScheduleStage {
     fn ensure_unlock_for_failed_execution(
         address_book: &mut AddressBook,
         lock_attempts: &mut Vec<LockAttempt>,
+        from_runnable: bool,
     ) {
         for l in lock_attempts {
             address_book.ensure_unlock(l);
 
             // revert because now contended again
-            //error!("n u a len() before: {}", address_book.newly_uncontended_addresses.len());
-            address_book.newly_uncontended_addresses.remove(&l.target);
-            //error!("n u a len() after: {}", address_book.newly_uncontended_addresses.len());
+            if !from_runnable {
+                //error!("n u a len() before: {}", address_book.newly_uncontended_addresses.len());
+                address_book.newly_uncontended_addresses.remove(&l.target);
+                //error!("n u a len() after: {}", address_book.newly_uncontended_addresses.len());
+            }
 
             // todo: mem::forget and panic in LockAttempt::drop()
         }

From b9b2d1e1a9fe85d46478cbe79fe2339f023534f7 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 9 Aug 2022 11:22:25 +0900
Subject: [PATCH 0402/3199] save

---
 scheduler/src/lib.rs | 16 +++++++++-------
 1 file changed, 9 insertions(+), 7 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 663396ae23b282..40481745829e9d 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -162,7 +162,7 @@ impl Page {
 //type AddressMap = std::collections::HashMap<Pubkey, PageRc>;
 type AddressMap = std::sync::Arc<dashmap::DashMap<Pubkey, PageRc>>;
 use by_address::ByAddress;
-type AddressSet = std::collections::HashSet<PageRc>;
+type AddressSet = std::collections::BTreeMap<(UniqueWeight, Pubkey), PageRc>;
 //type AddressMapEntry<'a, K, V> = std::collections::hash_map::Entry<'a, K, V>;
 type AddressMapEntry<'a> = dashmap::mapref::entry::Entry<'a, Pubkey, PageRc>;
 
@@ -454,7 +454,7 @@ impl ScheduleStage {
     fn get_weight_from_contended(address_book: &AddressBook) -> Option<UniqueWeight> {
         let mut heaviest_weight: Option<UniqueWeight> = None;
         trace!("n_u_a len(): {}", address_book.newly_uncontended_addresses.len());
-        for page in address_book.newly_uncontended_addresses.iter() {
+        for page in address_book.newly_uncontended_addresses.last() {
             let newly_uncontended_unique_weights = &page.0.contended_unique_weights;
             if let Some(&weight) = newly_uncontended_unique_weights.last() {
                 if let Some(current_heaviest_weight) = heaviest_weight {
@@ -545,6 +545,7 @@ impl ScheduleStage {
                 //trace!("ensure_unlock_for_failed_execution(): {:?} {}", (&unique_weight, from_runnable), next_task.tx.0.signature());
                 Self::ensure_unlock_for_failed_execution(
                     address_book,
+                &unique_weight,
                     &mut populated_lock_attempts,
                     from_runnable,
                 );
@@ -586,13 +587,14 @@ impl ScheduleStage {
             address_book.forget_address_contention(&unique_weight, &mut l);
 
             // revert because now contended again
-            address_book.newly_uncontended_addresses.remove(&l.target);
+            address_book.newly_uncontended_addresses.remove(&(unique_weight, l.target));
         }
     }
 
     #[inline(never)]
     fn ensure_unlock_for_failed_execution(
         address_book: &mut AddressBook,
+        unique_weight: &UniqueWeight,
         lock_attempts: &mut Vec<LockAttempt>,
         from_runnable: bool,
     ) {
@@ -602,7 +604,7 @@ impl ScheduleStage {
             // revert because now contended again
             if !from_runnable {
                 //error!("n u a len() before: {}", address_book.newly_uncontended_addresses.len());
-                address_book.newly_uncontended_addresses.remove(&l.target);
+                address_book.newly_uncontended_addresses.remove(&(unique_weight, l.target));
                 //error!("n u a len() after: {}", address_book.newly_uncontended_addresses.len());
             }
 
@@ -611,11 +613,11 @@ impl ScheduleStage {
     }
 
     #[inline(never)]
-    fn unlock_after_execution(address_book: &mut AddressBook, lock_attempts: Vec<LockAttempt>) {
+    fn unlock_after_execution(address_book: &mut AddressBook, unique_weight: &UniqueWeight, lock_attempts: Vec<LockAttempt>) {
         for mut l in lock_attempts.into_iter() {
             let newly_uncontended_while_queued = address_book.unlock(&mut l);
             if newly_uncontended_while_queued {
-                address_book.newly_uncontended_addresses.insert(l.target);
+                address_book.newly_uncontended_addresses.insert((unique_weight, l.target);
             }
 
             // todo: mem::forget and panic in LockAttempt::drop()
@@ -643,7 +645,7 @@ impl ScheduleStage {
     fn commit_result(ee: &mut ExecutionEnvironment, address_book: &mut AddressBook) {
         let lock_attempts = std::mem::take(&mut ee.lock_attempts);
         // do par()-ly?
-        Self::unlock_after_execution(address_book, lock_attempts);
+        Self::unlock_after_execution(address_book, ee.unique_weight, lock_attempts);
         // block-wide qos validation will be done here
         // if error risen..:
         //   don't commit the tx for banking and potentially finish scheduling at block max cu

From 2d9f83fe3890d03b339578249e508a2ae8b2b91e Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 9 Aug 2022 11:28:53 +0900
Subject: [PATCH 0403/3199] save

---
 scheduler/src/lib.rs | 30 ++++++++++++++----------------
 1 file changed, 14 insertions(+), 16 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 40481745829e9d..e0b485be6079be 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -162,7 +162,7 @@ impl Page {
 //type AddressMap = std::collections::HashMap<Pubkey, PageRc>;
 type AddressMap = std::sync::Arc<dashmap::DashMap<Pubkey, PageRc>>;
 use by_address::ByAddress;
-type AddressSet = std::collections::BTreeMap<(UniqueWeight, Pubkey), PageRc>;
+type AddressSet = std::collections::BTreeSet<UniqueWeight>;
 //type AddressMapEntry<'a, K, V> = std::collections::hash_map::Entry<'a, K, V>;
 type AddressMapEntry<'a> = dashmap::mapref::entry::Entry<'a, Pubkey, PageRc>;
 
@@ -545,7 +545,6 @@ impl ScheduleStage {
                 //trace!("ensure_unlock_for_failed_execution(): {:?} {}", (&unique_weight, from_runnable), next_task.tx.0.signature());
                 Self::ensure_unlock_for_failed_execution(
                     address_book,
-                &unique_weight,
                     &mut populated_lock_attempts,
                     from_runnable,
                 );
@@ -585,39 +584,38 @@ impl ScheduleStage {
         for mut l in lock_attempts {
             // ensure to remove remaining refs of this unique_weight
             address_book.forget_address_contention(&unique_weight, &mut l);
-
-            // revert because now contended again
-            address_book.newly_uncontended_addresses.remove(&(unique_weight, l.target));
         }
+
+        // revert because now contended again
+        address_book.newly_uncontended_addresses.remove(&unique_weight);
     }
 
     #[inline(never)]
     fn ensure_unlock_for_failed_execution(
         address_book: &mut AddressBook,
-        unique_weight: &UniqueWeight,
         lock_attempts: &mut Vec<LockAttempt>,
         from_runnable: bool,
     ) {
         for l in lock_attempts {
             address_book.ensure_unlock(l);
 
-            // revert because now contended again
-            if !from_runnable {
-                //error!("n u a len() before: {}", address_book.newly_uncontended_addresses.len());
-                address_book.newly_uncontended_addresses.remove(&(unique_weight, l.target));
-                //error!("n u a len() after: {}", address_book.newly_uncontended_addresses.len());
-            }
-
             // todo: mem::forget and panic in LockAttempt::drop()
         }
+
+        // revert because now contended again
+        if !from_runnable {
+            //error!("n u a len() before: {}", address_book.newly_uncontended_addresses.len());
+            address_book.newly_uncontended_addresses.remove(&unique_weight);
+            //error!("n u a len() after: {}", address_book.newly_uncontended_addresses.len());
+        }
     }
 
     #[inline(never)]
-    fn unlock_after_execution(address_book: &mut AddressBook, unique_weight: &UniqueWeight, lock_attempts: Vec<LockAttempt>) {
+    fn unlock_after_execution(address_book: &mut AddressBook, lock_attempts: Vec<LockAttempt>) {
         for mut l in lock_attempts.into_iter() {
             let newly_uncontended_while_queued = address_book.unlock(&mut l);
             if newly_uncontended_while_queued {
-                address_book.newly_uncontended_addresses.insert((unique_weight, l.target);
+                address_book.newly_uncontended_addresses.insert(l.target.contended_unique_weights.last());
             }
 
             // todo: mem::forget and panic in LockAttempt::drop()
@@ -645,7 +643,7 @@ impl ScheduleStage {
     fn commit_result(ee: &mut ExecutionEnvironment, address_book: &mut AddressBook) {
         let lock_attempts = std::mem::take(&mut ee.lock_attempts);
         // do par()-ly?
-        Self::unlock_after_execution(address_book, ee.unique_weight, lock_attempts);
+        Self::unlock_after_execution(address_book, lock_attempts);
         // block-wide qos validation will be done here
         // if error risen..:
         //   don't commit the tx for banking and potentially finish scheduling at block max cu

From 83720e14411ae6c2755ec51d6f940ba2a88e8d92 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 9 Aug 2022 11:30:03 +0900
Subject: [PATCH 0404/3199] save

---
 scheduler/src/lib.rs | 13 +++++--------
 1 file changed, 5 insertions(+), 8 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index e0b485be6079be..2ea178462c9674 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -454,16 +454,13 @@ impl ScheduleStage {
     fn get_weight_from_contended(address_book: &AddressBook) -> Option<UniqueWeight> {
         let mut heaviest_weight: Option<UniqueWeight> = None;
         trace!("n_u_a len(): {}", address_book.newly_uncontended_addresses.len());
-        for page in address_book.newly_uncontended_addresses.last() {
-            let newly_uncontended_unique_weights = &page.0.contended_unique_weights;
-            if let Some(&weight) = newly_uncontended_unique_weights.last() {
-                if let Some(current_heaviest_weight) = heaviest_weight {
-                    if weight > current_heaviest_weight {
-                        heaviest_weight = Some(weight);
-                    }
-                } else {
+        if let Some(&weight) = address_book.newly_uncontended_addresses.last() {
+            if let Some(current_heaviest_weight) = heaviest_weight {
+                if weight > current_heaviest_weight {
                     heaviest_weight = Some(weight);
                 }
+            } else {
+                heaviest_weight = Some(weight);
             }
         }
         heaviest_weight

From d3f0c879a6a0f237d685606a8d42349257a1e245 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 9 Aug 2022 11:31:03 +0900
Subject: [PATCH 0405/3199] save

---
 scheduler/src/lib.rs | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 2ea178462c9674..ddf3fa24a68ea6 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -542,6 +542,7 @@ impl ScheduleStage {
                 //trace!("ensure_unlock_for_failed_execution(): {:?} {}", (&unique_weight, from_runnable), next_task.tx.0.signature());
                 Self::ensure_unlock_for_failed_execution(
                     address_book,
+                    &unique_weight,
                     &mut populated_lock_attempts,
                     from_runnable,
                 );
@@ -590,6 +591,7 @@ impl ScheduleStage {
     #[inline(never)]
     fn ensure_unlock_for_failed_execution(
         address_book: &mut AddressBook,
+        unique_weight: &UniqueWeight,
         lock_attempts: &mut Vec<LockAttempt>,
         from_runnable: bool,
     ) {

From 672a131d4710e25f9c3d6cc36397a87176e34230 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 9 Aug 2022 11:31:29 +0900
Subject: [PATCH 0406/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index ddf3fa24a68ea6..19122236599805 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -614,7 +614,7 @@ impl ScheduleStage {
         for mut l in lock_attempts.into_iter() {
             let newly_uncontended_while_queued = address_book.unlock(&mut l);
             if newly_uncontended_while_queued {
-                address_book.newly_uncontended_addresses.insert(l.target.contended_unique_weights.last());
+                address_book.newly_uncontended_addresses.insert(l.target.page().contended_unique_weights.last());
             }
 
             // todo: mem::forget and panic in LockAttempt::drop()

From be456bbbc38cdff7a876e7df32e176d73dbd7967 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 9 Aug 2022 11:32:07 +0900
Subject: [PATCH 0407/3199] save

---
 scheduler/src/lib.rs | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 19122236599805..ac62b3d68ca392 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -614,7 +614,9 @@ impl ScheduleStage {
         for mut l in lock_attempts.into_iter() {
             let newly_uncontended_while_queued = address_book.unlock(&mut l);
             if newly_uncontended_while_queued {
-                address_book.newly_uncontended_addresses.insert(l.target.page().contended_unique_weights.last());
+                if let Some(uw) = l.target.page().contended_unique_weights.last() {
+                    address_book.newly_uncontended_addresses.insert(uw);
+                }
             }
 
             // todo: mem::forget and panic in LockAttempt::drop()

From 655a800843c4a842f450591864991cc593d21ade Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 9 Aug 2022 11:32:18 +0900
Subject: [PATCH 0408/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index ac62b3d68ca392..a2e760473349cf 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -615,7 +615,7 @@ impl ScheduleStage {
             let newly_uncontended_while_queued = address_book.unlock(&mut l);
             if newly_uncontended_while_queued {
                 if let Some(uw) = l.target.page().contended_unique_weights.last() {
-                    address_book.newly_uncontended_addresses.insert(uw);
+                    address_book.newly_uncontended_addresses.insert(*uw);
                 }
             }
 

From 7ce18ab324891ede6bff94c8759eecb7d79f2864 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 9 Aug 2022 18:12:54 +0900
Subject: [PATCH 0409/3199] save

---
 scheduler/src/lib.rs | 12 +-----------
 1 file changed, 1 insertion(+), 11 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index a2e760473349cf..f552c057ebe597 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -452,18 +452,8 @@ impl ScheduleStage {
 
     #[inline(never)]
     fn get_weight_from_contended(address_book: &AddressBook) -> Option<UniqueWeight> {
-        let mut heaviest_weight: Option<UniqueWeight> = None;
         trace!("n_u_a len(): {}", address_book.newly_uncontended_addresses.len());
-        if let Some(&weight) = address_book.newly_uncontended_addresses.last() {
-            if let Some(current_heaviest_weight) = heaviest_weight {
-                if weight > current_heaviest_weight {
-                    heaviest_weight = Some(weight);
-                }
-            } else {
-                heaviest_weight = Some(weight);
-            }
-        }
-        heaviest_weight
+        address_book.newly_uncontended_addresses.last()
     }
 
     #[inline(never)]

From 733c365ad481ad29a2f7aa2bfab808359f9a7223 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 9 Aug 2022 18:14:41 +0900
Subject: [PATCH 0410/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index f552c057ebe597..be0b5a368bca93 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -453,7 +453,7 @@ impl ScheduleStage {
     #[inline(never)]
     fn get_weight_from_contended(address_book: &AddressBook) -> Option<UniqueWeight> {
         trace!("n_u_a len(): {}", address_book.newly_uncontended_addresses.len());
-        address_book.newly_uncontended_addresses.last()
+        address_book.newly_uncontended_addresses.last().map(|w| *w)
     }
 
     #[inline(never)]

From e9401e51d27fa14674182996188b5ff9baca5a83 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 9 Aug 2022 18:19:59 +0900
Subject: [PATCH 0411/3199] save

---
 transaction-scheduler-bench/src/main.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/transaction-scheduler-bench/src/main.rs b/transaction-scheduler-bench/src/main.rs
index 8db94f59350594..f1ad2602aee6e4 100644
--- a/transaction-scheduler-bench/src/main.rs
+++ b/transaction-scheduler-bench/src/main.rs
@@ -345,7 +345,7 @@ fn handle_transaction_batch(
             .unwrap();
 }
 
-const NUM_SENDERS: usize = 1;
+const NUM_SENDERS: usize = 2;
 
 fn spawn_packet_senders(
     preloader: Arc<solana_scheduler::Preloader>,

From 6810f4161e1670a27ab6b7fe82614f7aadea9aa0 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 9 Aug 2022 18:27:16 +0900
Subject: [PATCH 0412/3199] save

---
 scheduler/src/lib.rs | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index be0b5a368bca93..328b9f3d89501b 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -587,6 +587,9 @@ impl ScheduleStage {
     ) {
         for l in lock_attempts {
             address_book.ensure_unlock(l);
+            if let Some(uw) = l.target.page().contended_unique_weights.last() {
+                address_book.newly_uncontended_addresses.remove(*uw);
+            }
 
             // todo: mem::forget and panic in LockAttempt::drop()
         }

From 90b2c451257b4afdea4363be5bde9c177349d971 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 9 Aug 2022 18:27:28 +0900
Subject: [PATCH 0413/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 328b9f3d89501b..aba41837c6df0b 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -588,7 +588,7 @@ impl ScheduleStage {
         for l in lock_attempts {
             address_book.ensure_unlock(l);
             if let Some(uw) = l.target.page().contended_unique_weights.last() {
-                address_book.newly_uncontended_addresses.remove(*uw);
+                address_book.newly_uncontended_addresses.remove(uw);
             }
 
             // todo: mem::forget and panic in LockAttempt::drop()

From 4a2b80c207e2905cbe38412fe7d8abbd0fd75551 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 9 Aug 2022 18:30:23 +0900
Subject: [PATCH 0414/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index aba41837c6df0b..543008d3e89d0f 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -549,7 +549,7 @@ impl ScheduleStage {
                 continue;
             }
 
-            trace!("successful lock");
+            trace!("successful lock: (from_runnable: {})", from_runnable);
             Self::finalize_successful_lock_before_execution(
                 address_book,
                 &unique_weight,

From d9e8f1a9574412cae4a1681bce8e0da69005f5b5 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 9 Aug 2022 18:36:42 +0900
Subject: [PATCH 0415/3199] save

---
 scheduler/src/lib.rs | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 543008d3e89d0f..c766f93219506e 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -498,7 +498,10 @@ impl ScheduleStage {
                     )
                 }
             }
-            (None, None) => None,
+            (None, None) => {
+                trace!("select: none");
+                None
+            }
         }
     }
 

From 79da1d3925156102a0cb55d45ad9d44e86676043 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 9 Aug 2022 18:38:58 +0900
Subject: [PATCH 0416/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index c766f93219506e..664f97d8463fb8 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -744,7 +744,7 @@ impl ScheduleStage {
 
                     to_execute_substage.send(ee).unwrap();
                 } else {
-                    trace!("incoming queue starved");
+                    trace!("incoming queue starved: n_u_a: {}", address_book.newly_uncontended_addresses.len());
                     break;
                 }
             }

From b8286b7c0da0ee8646abb7960239599f3bb7acb9 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 9 Aug 2022 18:43:14 +0900
Subject: [PATCH 0417/3199] save

---
 scheduler/src/lib.rs | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 664f97d8463fb8..f1fbfb517f43a2 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -511,9 +511,11 @@ impl ScheduleStage {
         contended_queue: &mut TaskQueue,
         address_book: &mut AddressBook,
     ) -> Option<(UniqueWeight, Task, Vec<LockAttempt>)> {
+        trace!("pop begin");
         for (reborrowed_contended_queue, mut queue_entry) in
             Self::select_next_task(runnable_queue, contended_queue, address_book)
         {
+            trace!("pop loop iteration");
             let from_runnable = reborrowed_contended_queue.is_some();
             let unique_weight = *queue_entry.key();
             let next_task = queue_entry.get_mut();

From bb3907707fc85ebd0b5e4a43e37692db8a7b170c Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 9 Aug 2022 18:45:51 +0900
Subject: [PATCH 0418/3199] save

---
 scheduler/src/lib.rs | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index f1fbfb517f43a2..6eac412d94b5fd 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -512,9 +512,8 @@ impl ScheduleStage {
         address_book: &mut AddressBook,
     ) -> Option<(UniqueWeight, Task, Vec<LockAttempt>)> {
         trace!("pop begin");
-        for (reborrowed_contended_queue, mut queue_entry) in
-            Self::select_next_task(runnable_queue, contended_queue, address_book)
-        {
+        loop {
+        if let Some(reborrowed_contended_queue, mut queue_entry) = Self::select_next_task(runnable_queue, contended_queue, address_book) {
             trace!("pop loop iteration");
             let from_runnable = reborrowed_contended_queue.is_some();
             let unique_weight = *queue_entry.key();
@@ -562,6 +561,9 @@ impl ScheduleStage {
             );
             let task = queue_entry.remove();
             return Some((unique_weight, task, populated_lock_attempts));
+        } else {
+            break;
+        }
         }
 
         None

From 4b854e58548cd455642ac997c85aa0f9fa586400 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 9 Aug 2022 18:46:16 +0900
Subject: [PATCH 0419/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 6eac412d94b5fd..c5edd75e65ef0c 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -513,7 +513,7 @@ impl ScheduleStage {
     ) -> Option<(UniqueWeight, Task, Vec<LockAttempt>)> {
         trace!("pop begin");
         loop {
-        if let Some(reborrowed_contended_queue, mut queue_entry) = Self::select_next_task(runnable_queue, contended_queue, address_book) {
+        if let Some((reborrowed_contended_queue, mut queue_entry)) = Self::select_next_task(runnable_queue, contended_queue, address_book) {
             trace!("pop loop iteration");
             let from_runnable = reborrowed_contended_queue.is_some();
             let unique_weight = *queue_entry.key();

From 2bb349c1bfbb6899a2690e7205f802b03c9335d7 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 9 Aug 2022 18:55:13 +0900
Subject: [PATCH 0420/3199] save

---
 scheduler/src/lib.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index c5edd75e65ef0c..23424c12f50bb5 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -732,9 +732,9 @@ impl ScheduleStage {
             }
 
             loop {
-                if !address_book.newly_uncontended_addresses.is_empty() {
+                /*if !address_book.newly_uncontended_addresses.is_empty() {
                     trace!("prefer emptying n_u_a");
-                } else if executing_queue_count >= max_executing_queue_count {
+                } else */ if executing_queue_count >= max_executing_queue_count {
                     trace!("outgoing queue full");
                     break;
                 }

From b068e9abff52a7c9771ff78da1edfe88bb7df196 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 9 Aug 2022 19:15:04 +0900
Subject: [PATCH 0421/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 23424c12f50bb5..5ec7a0812335d7 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -548,7 +548,7 @@ impl ScheduleStage {
                         .add_to_schedule(*queue_entry.key(), queue_entry.remove());
                     // maybe run lightweight prune logic on contended_queue here.
                 } else {
-                    trace!("relock failed; remains in contended");
+                    trace!("relock failed; remains in contended: {}", unique_weight);
                 }
                 continue;
             }

From 6ae9adbf38a4bd36700f812f861323e02bfa087e Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 9 Aug 2022 19:15:25 +0900
Subject: [PATCH 0422/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 5ec7a0812335d7..b881b7684f5da5 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -548,7 +548,7 @@ impl ScheduleStage {
                         .add_to_schedule(*queue_entry.key(), queue_entry.remove());
                     // maybe run lightweight prune logic on contended_queue here.
                 } else {
-                    trace!("relock failed; remains in contended: {}", unique_weight);
+                    trace!("relock failed; remains in contended: {}", &unique_weight);
                 }
                 continue;
             }

From 27fc34ffb6687833dd9203adc222e1e76dbe7bb9 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 9 Aug 2022 19:15:37 +0900
Subject: [PATCH 0423/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index b881b7684f5da5..fbaf81444e199e 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -548,7 +548,7 @@ impl ScheduleStage {
                         .add_to_schedule(*queue_entry.key(), queue_entry.remove());
                     // maybe run lightweight prune logic on contended_queue here.
                 } else {
-                    trace!("relock failed; remains in contended: {}", &unique_weight);
+                    trace!("relock failed; remains in contended: {:?}", &unique_weight);
                 }
                 continue;
             }

From 6d560b9b57338dd53b398c9da707ae0544848f32 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 9 Aug 2022 19:21:04 +0900
Subject: [PATCH 0424/3199] save

---
 scheduler/src/lib.rs | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index fbaf81444e199e..d21d2e44229050 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -594,9 +594,9 @@ impl ScheduleStage {
     ) {
         for l in lock_attempts {
             address_book.ensure_unlock(l);
-            if let Some(uw) = l.target.page().contended_unique_weights.last() {
-                address_book.newly_uncontended_addresses.remove(uw);
-            }
+            //if let Some(uw) = l.target.page().contended_unique_weights.last() {
+            //    address_book.newly_uncontended_addresses.remove(uw);
+            //}
 
             // todo: mem::forget and panic in LockAttempt::drop()
         }

From 8bed957755953ea1627fc4df2252c50a1d3ff300 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 9 Aug 2022 21:53:16 +0900
Subject: [PATCH 0425/3199] save

---
 scheduler/src/lib.rs | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index d21d2e44229050..4fd1aa780816e9 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -541,6 +541,8 @@ impl ScheduleStage {
                     from_runnable,
                 );
                 std::mem::swap(&mut next_task.tx.1, &mut populated_lock_attempts);
+                next_task.contention_count += 1;
+
                 if from_runnable {
                     trace!("move to contended due to lock failure");
                     reborrowed_contended_queue
@@ -548,12 +550,12 @@ impl ScheduleStage {
                         .add_to_schedule(*queue_entry.key(), queue_entry.remove());
                     // maybe run lightweight prune logic on contended_queue here.
                 } else {
-                    trace!("relock failed; remains in contended: {:?}", &unique_weight);
+                    trace!("relock failed; remains in contended: {:?} contention: {}", &unique_weight, next_task.contention_count);
                 }
                 continue;
             }
 
-            trace!("successful lock: (from_runnable: {})", from_runnable);
+            trace!("successful lock: (from_runnable: {}) after {} contentions", from_runnable, next_task.contention_count);
             Self::finalize_successful_lock_before_execution(
                 address_book,
                 &unique_weight,

From 8cf9493dd2ddca63f5e7d80119f9b8440de8af24 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 9 Aug 2022 21:53:41 +0900
Subject: [PATCH 0426/3199] save

---
 scheduler/src/lib.rs | 1 +
 1 file changed, 1 insertion(+)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 4fd1aa780816e9..be3b7e1b80cba0 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -323,6 +323,7 @@ struct Bundle {
 #[derive(Debug)]
 pub struct Task {
     pub tx: Box<(SanitizedTransaction, Vec<LockAttempt>)>, // actually should be Bundle
+    pub contention_count: usize,
 }
 
 // RunnableQueue, ContendedQueue?

From cd7a588d6e6df4bf3966eb62105894425612ce3a Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 9 Aug 2022 21:53:53 +0900
Subject: [PATCH 0427/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index be3b7e1b80cba0..7872e478077d56 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -431,7 +431,7 @@ impl ScheduleStage {
                 //unique_key: solana_sdk::hash::new_rand(&mut rng),
                 unique_key: *unique_key,
             },
-            Task { tx },
+            Task { tx, contention_count: 0 },
         );
         *unique_key -= 1;
     }

From 698a1fb7117f234149a577fe8d5f72adec1063a0 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 9 Aug 2022 21:58:01 +0900
Subject: [PATCH 0428/3199] save

---
 scheduler/src/lib.rs | 19 ++++++++++---------
 1 file changed, 10 insertions(+), 9 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 7872e478077d56..976b47f5d6ba56 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -369,18 +369,18 @@ fn attempt_lock_for_execution<'a>(
     unique_weight: &UniqueWeight,
     message_hash: &'a Hash,
     mut placeholder_attempts: Vec<LockAttempt>,
-) -> (bool, Vec<LockAttempt>) {
+) -> (usize, Vec<LockAttempt>) {
     // no short-cuircuit; we at least all need to add to the contended queue
-    let mut all_succeeded_so_far = true;
+    let mut unlockable_count = 0;
 
     for attempt in placeholder_attempts.iter_mut() {
         address_book.attempt_lock_address(from_runnable, unique_weight, attempt);
-        if all_succeeded_so_far && attempt.is_failed() {
-            all_succeeded_so_far = false;
+        if attempt.is_failed() {
+            unlockable_count += 1;
         }
     }
 
-    (all_succeeded_so_far, placeholder_attempts)
+    (unlockable_count, placeholder_attempts)
 }
 
 type PreprocessedTransaction = (SanitizedTransaction, Vec<LockAttempt>);
@@ -525,7 +525,7 @@ impl ScheduleStage {
             // plumb message_hash into StatusCache or implmenent our own for duplicate tx
             // detection?
 
-            let (is_success, mut populated_lock_attempts) = attempt_lock_for_execution(
+            let (unlockable_count, mut populated_lock_attempts) = attempt_lock_for_execution(
                 from_runnable,
                 address_book,
                 &unique_weight,
@@ -533,7 +533,7 @@ impl ScheduleStage {
                 placeholder_lock_attempts,
             );
 
-            if !is_success {
+            if unlockable_count > 0 {
                 //trace!("ensure_unlock_for_failed_execution(): {:?} {}", (&unique_weight, from_runnable), next_task.tx.0.signature());
                 Self::ensure_unlock_for_failed_execution(
                     address_book,
@@ -541,17 +541,18 @@ impl ScheduleStage {
                     &mut populated_lock_attempts,
                     from_runnable,
                 );
+                let lock_count = populated_lock_attempts.len();
                 std::mem::swap(&mut next_task.tx.1, &mut populated_lock_attempts);
                 next_task.contention_count += 1;
 
                 if from_runnable {
-                    trace!("move to contended due to lock failure");
+                    trace!("move to contended due to lock failure [{}/{}]", unlockable_count, lock_count);
                     reborrowed_contended_queue
                         .unwrap()
                         .add_to_schedule(*queue_entry.key(), queue_entry.remove());
                     // maybe run lightweight prune logic on contended_queue here.
                 } else {
-                    trace!("relock failed; remains in contended: {:?} contention: {}", &unique_weight, next_task.contention_count);
+                    trace!("relock failed [{}/{}]; remains in contended: {:?} contention: {}", unlockable_count, lock_count, &unique_weight, next_task.contention_count);
                 }
                 continue;
             }

From 24a6382099f556ad6ef62efbbf060adc9f463e3e Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 9 Aug 2022 22:03:55 +0900
Subject: [PATCH 0429/3199] save

---
 ledger-tool/src/main.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/ledger-tool/src/main.rs b/ledger-tool/src/main.rs
index 15463aa9a65988..3a500b49dfba88 100644
--- a/ledger-tool/src/main.rs
+++ b/ledger-tool/src/main.rs
@@ -328,7 +328,7 @@ fn output_slot(
                         }
 
                         muxed_sender
-                            .send(solana_scheduler::MultiplexedPayload::FromExecute(ee))
+                            .send(solana_scheduler::Multiplexed::FromExecute(ee))
                             .unwrap();
                     }
                 })
@@ -383,7 +383,7 @@ fn output_slot(
                 }
 
                 muxed_sender
-                    .send(solana_scheduler::MultiplexedPayload::FromPrevious((
+                    .send(solana_scheduler::Multiplexed::FromPrevious((
                         Weight { ix: weight },
                         tx,
                     )))

From 1f4c2ff264ea2aafcc5e82d496cd9f0ad5e13147 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 9 Aug 2022 22:32:59 +0900
Subject: [PATCH 0430/3199] save

---
 scheduler/src/lib.rs | 10 +++++++++-
 1 file changed, 9 insertions(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 976b47f5d6ba56..bf072945526e86 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -84,10 +84,17 @@ impl PageRc {
     }
 }
 
+#[derive(Clone, Debug)]
+enum LockStatus {
+    Succeded,
+    Guaranteed, 
+    Failed,
+}
+
 #[derive(Clone, Debug)]
 pub struct LockAttempt {
     target: PageRc,
-    is_success: bool,
+    status: LockStatus,
     requested_usage: RequestedUsage,
 }
 
@@ -144,6 +151,7 @@ pub enum RequestedUsage {
 struct Page {
     current_usage: CurrentUsage,
     contended_unique_weights: std::collections::BTreeSet<UniqueWeight>,
+    next_usage: Option<CurrentUsage>,
     //next_scheduled_task // reserved_task guaranteed_task
     //loaded account from Accounts db
     //comulative_cu for qos; i.e. track serialized cumulative keyed by addresses and bail out block

From 4a7eed311773b9d097a8228b0ff02e5e69aa17ec Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 9 Aug 2022 22:34:25 +0900
Subject: [PATCH 0431/3199] save

---
 scheduler/src/lib.rs | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index bf072945526e86..b05e61f9217738 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -189,25 +189,25 @@ impl AddressBook {
         unique_weight: &UniqueWeight,
         attempt: &mut LockAttempt,
     ) {
-        let LockAttempt {target, requested_usage, is_success} = attempt;
+        let LockAttempt {target, requested_usage, status} = attempt;
 
                 let mut page = unsafe { MyRcInner::get_mut_unchecked(&mut target.0) };
 
                 match page.current_usage {
                     CurrentUsage::Unused => {
                         page.current_usage = CurrentUsage::renew(*requested_usage);
-                        *is_success = true;
+                        *status = LockStatus::Succeded;
                     }
                     CurrentUsage::Readonly(ref mut count) => match requested_usage {
                         RequestedUsage::Readonly => {
                             *count += 1;
-                            *is_success = true;
+                            *status = LockStatus::Succeded;
                         }
                         RequestedUsage::Writable => {
                             if from_runnable {
                                 Self::remember_address_contention(&mut page, unique_weight);
                             }
-                            *is_success = false;
+                            *status = LockStatus::Failed;
                         }
                     },
                     CurrentUsage::Writable => match requested_usage {
@@ -215,7 +215,7 @@ impl AddressBook {
                             if from_runnable {
                                 Self::remember_address_contention(&mut page, unique_weight);
                             }
-                            *is_success = false;
+                            *status = LockStatus::Failed;
                         }
                     },
                 }

From 8346f835a5c46d70c837e6979630794be4eac4b6 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 9 Aug 2022 22:38:09 +0900
Subject: [PATCH 0432/3199] save

---
 scheduler/src/lib.rs | 12 +++++++++++-
 1 file changed, 11 insertions(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index b05e61f9217738..3a9736ca80da1b 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -206,8 +206,18 @@ impl AddressBook {
                         RequestedUsage::Writable => {
                             if from_runnable {
                                 Self::remember_address_contention(&mut page, unique_weight);
+                                *status = LockStatus::Failed;
+                            } else {
+                                match page.next_usage {
+                                    CurrentUsage::Unused => {
+                                        *status = LockStatus::Guaranteed;
+                                        page.next_usage = CurrentUsage::renew(*requested_usage);
+                                    },
+                                    CurrentUsage::Readonly | CurrentUsage::Writable {
+                                        *status = LockStatus::Failed;
+                                    },
+                                }
                             }
-                            *status = LockStatus::Failed;
                         }
                     },
                     CurrentUsage::Writable => match requested_usage {

From a921ce0bb0736114686993f3be4a5c6814b0f102 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 9 Aug 2022 22:38:31 +0900
Subject: [PATCH 0433/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 3a9736ca80da1b..853735074e4a03 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -213,7 +213,7 @@ impl AddressBook {
                                         *status = LockStatus::Guaranteed;
                                         page.next_usage = CurrentUsage::renew(*requested_usage);
                                     },
-                                    CurrentUsage::Readonly | CurrentUsage::Writable {
+                                    CurrentUsage::Readonly | CurrentUsage::Writable => {
                                         *status = LockStatus::Failed;
                                     },
                                 }

From 297282daab5d6a54afe6c624a9ac57755eb75a47 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 9 Aug 2022 22:40:08 +0900
Subject: [PATCH 0434/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 853735074e4a03..9bcc006698e1fa 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -213,7 +213,7 @@ impl AddressBook {
                                         *status = LockStatus::Guaranteed;
                                         page.next_usage = CurrentUsage::renew(*requested_usage);
                                     },
-                                    CurrentUsage::Readonly | CurrentUsage::Writable => {
+                                    CurrentUsage::Readonly(_) | CurrentUsage::Writable => {
                                         *status = LockStatus::Failed;
                                     },
                                 }

From 64e6d49c69e6718d43e655cdae8595d66360ca90 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 9 Aug 2022 22:57:56 +0900
Subject: [PATCH 0435/3199] save

---
 scheduler/src/lib.rs | 77 ++++++++++++++++++++++++++++++--------------
 1 file changed, 52 insertions(+), 25 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 9bcc006698e1fa..ce4125ecc4da93 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -120,7 +120,7 @@ type UsageCount = usize;
 const SOLE_USE_COUNT: UsageCount = 1;
 
 #[derive(Debug, PartialEq)]
-enum CurrentUsage {
+enum Usage {
     Unused,
     // weight to abort running tx?
     // also sum all readonly weights to subvert to write lock with greater weight?
@@ -128,16 +128,16 @@ enum CurrentUsage {
     Writable,
 }
 
-impl CurrentUsage {
+impl Usage {
     fn renew(requested_usage: RequestedUsage) -> Self {
         match requested_usage {
-            RequestedUsage::Readonly => CurrentUsage::Readonly(SOLE_USE_COUNT),
-            RequestedUsage::Writable => CurrentUsage::Writable,
+            RequestedUsage::Readonly => Usage::Readonly(SOLE_USE_COUNT),
+            RequestedUsage::Writable => Usage::Writable,
         }
     }
 
     fn unused() -> Self {
-        CurrentUsage::Unused
+        Usage::Unused
     }
 }
 
@@ -149,9 +149,9 @@ pub enum RequestedUsage {
 
 #[derive(Debug)]
 struct Page {
-    current_usage: CurrentUsage,
+    current_usage: Usage,
     contended_unique_weights: std::collections::BTreeSet<UniqueWeight>,
-    next_usage: Option<CurrentUsage>,
+    next_usage: Usage,
     //next_scheduled_task // reserved_task guaranteed_task
     //loaded account from Accounts db
     //comulative_cu for qos; i.e. track serialized cumulative keyed by addresses and bail out block
@@ -159,7 +159,7 @@ struct Page {
 }
 
 impl Page {
-    fn new(current_usage: CurrentUsage) -> Self {
+    fn new(current_usage: Usage) -> Self {
         Self {
             current_usage,
             contended_unique_weights: Default::default(),
@@ -194,11 +194,11 @@ impl AddressBook {
                 let mut page = unsafe { MyRcInner::get_mut_unchecked(&mut target.0) };
 
                 match page.current_usage {
-                    CurrentUsage::Unused => {
-                        page.current_usage = CurrentUsage::renew(*requested_usage);
+                    Usage::Unused => {
+                        page.current_usage = Usage::renew(*requested_usage);
                         *status = LockStatus::Succeded;
                     }
-                    CurrentUsage::Readonly(ref mut count) => match requested_usage {
+                    Usage::Readonly(ref mut count) => match requested_usage {
                         RequestedUsage::Readonly => {
                             *count += 1;
                             *status = LockStatus::Succeded;
@@ -209,18 +209,18 @@ impl AddressBook {
                                 *status = LockStatus::Failed;
                             } else {
                                 match page.next_usage {
-                                    CurrentUsage::Unused => {
+                                    Usage::Unused => {
                                         *status = LockStatus::Guaranteed;
-                                        page.next_usage = CurrentUsage::renew(*requested_usage);
+                                        page.next_usage = Usage::renew(*requested_usage);
                                     },
-                                    CurrentUsage::Readonly(_) | CurrentUsage::Writable => {
+                                    Usage::Readonly(_) | Usage::Writable => {
                                         *status = LockStatus::Failed;
                                     },
                                 }
                             }
                         }
                     },
-                    CurrentUsage::Writable => match requested_usage {
+                    Usage::Writable => match requested_usage {
                         RequestedUsage::Readonly | RequestedUsage::Writable => {
                             if from_runnable {
                                 Self::remember_address_contention(&mut page, unique_weight);
@@ -257,7 +257,7 @@ impl AddressBook {
         let mut page = attempt.target.page();
 
         match &mut page.current_usage {
-            CurrentUsage::Readonly(ref mut count) => match &attempt.requested_usage {
+            Usage::Readonly(ref mut count) => match &attempt.requested_usage {
                 RequestedUsage::Readonly => {
                     if *count == SOLE_USE_COUNT {
                         newly_uncontended = true;
@@ -267,17 +267,17 @@ impl AddressBook {
                 }
                 RequestedUsage::Writable => unreachable!(),
             },
-            CurrentUsage::Writable => match &attempt.requested_usage {
+            Usage::Writable => match &attempt.requested_usage {
                 RequestedUsage::Writable => {
                     newly_uncontended = true;
                 }
                 RequestedUsage::Readonly => unreachable!(),
             },
-            CurrentUsage::Unused => unreachable!(),
+            Usage::Unused => unreachable!(),
         }
 
         if newly_uncontended {
-            page.current_usage = CurrentUsage::Unused;
+            page.current_usage = Usage::Unused;
             if !page.contended_unique_weights.is_empty() {
                 still_queued = true;
             }
@@ -299,7 +299,7 @@ impl Preloader {
     pub fn load(&self, address: Pubkey) -> PageRc {
         match self.book.entry(address) {
             AddressMapEntry::Vacant(book_entry) => {
-                let page = PageRc(ByAddress(MyRcInner::new(Page::new(CurrentUsage::unused()))));
+                let page = PageRc(ByAddress(MyRcInner::new(Page::new(Usage::unused()))));
                 let cloned = PageRc::clone(&page);
                 book_entry.insert(page);
                 cloned
@@ -387,18 +387,25 @@ fn attempt_lock_for_execution<'a>(
     unique_weight: &UniqueWeight,
     message_hash: &'a Hash,
     mut placeholder_attempts: Vec<LockAttempt>,
-) -> (usize, Vec<LockAttempt>) {
+) -> (usize, usize, Vec<LockAttempt>) {
     // no short-cuircuit; we at least all need to add to the contended queue
     let mut unlockable_count = 0;
+    let mut guaranteed_count = 0;
 
     for attempt in placeholder_attempts.iter_mut() {
         address_book.attempt_lock_address(from_runnable, unique_weight, attempt);
-        if attempt.is_failed() {
-            unlockable_count += 1;
+        match address_book.status {
+            LockStatus::Succeded => {},
+            LockStatus::Failed => {
+                unlockable_count += 1;
+            },
+            LockStatus::Guaranteed => {
+                guaranteed_count += 1;
+            },
         }
     }
 
-    (unlockable_count, placeholder_attempts)
+    (unlockable_count, guaranteed_count, placeholder_attempts)
 }
 
 type PreprocessedTransaction = (SanitizedTransaction, Vec<LockAttempt>);
@@ -543,7 +550,7 @@ impl ScheduleStage {
             // plumb message_hash into StatusCache or implmenent our own for duplicate tx
             // detection?
 
-            let (unlockable_count, mut populated_lock_attempts) = attempt_lock_for_execution(
+            let (unlockable_count, guaranteed_count, mut populated_lock_attempts) = attempt_lock_for_execution(
                 from_runnable,
                 address_book,
                 &unique_weight,
@@ -572,6 +579,18 @@ impl ScheduleStage {
                 } else {
                     trace!("relock failed [{}/{}]; remains in contended: {:?} contention: {}", unlockable_count, lock_count, &unique_weight, next_task.contention_count);
                 }
+                continue;
+            } else if guaranteed_count > 0 {
+                assert!(!from_runnable);
+                address_book.guaranteed_locks.insert(unique_weight, guaranteed_count);
+                Self::ensure_unlock_for_guaranteed__execution(
+                    address_book,
+                    &unique_weight,
+                    &mut populated_lock_attempts,
+                    from_runnable,
+                );
+                std::mem::swap(&mut next_task.tx.1, &mut populated_lock_attempts);
+
                 continue;
             }
 
@@ -639,6 +658,14 @@ impl ScheduleStage {
                 if let Some(uw) = l.target.page().contended_unique_weights.last() {
                     address_book.newly_uncontended_addresses.insert(*uw);
                 }
+                for uw in l.target.page().guaranteed_unique_weights {
+                    if let Some(count) = address_book.guaranteed_locks.get_mut(&uw) {
+                        count -= 1;
+                        if count == 0 {
+                            address_book.runnable_guaranteed_unique_weights.insert(uw);
+                        }
+                    }
+                }
             }
 
             // todo: mem::forget and panic in LockAttempt::drop()

From a2333b2b06b3c32bb0ce6b1a5a7ec856419693fb Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 9 Aug 2022 22:58:14 +0900
Subject: [PATCH 0436/3199] save

---
 scheduler/src/lib.rs | 8 --------
 1 file changed, 8 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index ce4125ecc4da93..ff0f84924528fa 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -99,14 +99,6 @@ pub struct LockAttempt {
 }
 
 impl LockAttempt {
-    fn is_success(&self) -> bool {
-        self.is_success
-    }
-
-    fn is_failed(&self) -> bool {
-        !self.is_success()
-    }
-
     pub fn new(target: PageRc, requested_usage: RequestedUsage) -> Self {
         Self {
             target,

From fa0ddc16e8a25513de27c8b4b918e974d10de484 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 9 Aug 2022 22:58:36 +0900
Subject: [PATCH 0437/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index ff0f84924528fa..e5c1c6ec22d536 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -102,7 +102,7 @@ impl LockAttempt {
     pub fn new(target: PageRc, requested_usage: RequestedUsage) -> Self {
         Self {
             target,
-            is_success: true,
+            status: LockStatus::Succeded,
             requested_usage,
         }
     }

From cec09910d9f428e4ef2f5ff2d3312ff957175fda Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 9 Aug 2022 22:59:05 +0900
Subject: [PATCH 0438/3199] save

---
 scheduler/src/lib.rs | 1 +
 1 file changed, 1 insertion(+)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index e5c1c6ec22d536..6e2ce478eae4f4 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -154,6 +154,7 @@ impl Page {
     fn new(current_usage: Usage) -> Self {
         Self {
             current_usage,
+            next_usage: Usage::Unused,
             contended_unique_weights: Default::default(),
         }
     }

From b06ff8d063f4ad8dcc4d459b195eed00b7766eac Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Wed, 10 Aug 2022 10:35:00 +0900
Subject: [PATCH 0439/3199] save

---
 scheduler/src/lib.rs | 118 +++++++++++++++++++++++++++++++------------
 1 file changed, 87 insertions(+), 31 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 6e2ce478eae4f4..e3ad9c11b60592 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -144,7 +144,7 @@ struct Page {
     current_usage: Usage,
     contended_unique_weights: std::collections::BTreeSet<UniqueWeight>,
     next_usage: Usage,
-    //next_scheduled_task // reserved_task guaranteed_task
+    guaranteed_task_ids: WeightedTaskIds,
     //loaded account from Accounts db
     //comulative_cu for qos; i.e. track serialized cumulative keyed by addresses and bail out block
     //producing as soon as any one of cu from the executing thread reaches to the limit
@@ -158,12 +158,18 @@ impl Page {
             contended_unique_weights: Default::default(),
         }
     }
+
+    fn switch_to_next_usage(&mut self) {
+        self.current_usage = self.next_usage;
+        self.next_usage = Usage::Unused;
+    }
 }
 
 //type AddressMap = std::collections::HashMap<Pubkey, PageRc>;
 type AddressMap = std::sync::Arc<dashmap::DashMap<Pubkey, PageRc>>;
 use by_address::ByAddress;
-type AddressSet = std::collections::BTreeSet<UniqueWeight>;
+type TaskId = UniqueWeight;
+type WeightedTaskIds = std::collections::BTreeSet<TaskId>;
 //type AddressMapEntry<'a, K, V> = std::collections::hash_map::Entry<'a, K, V>;
 type AddressMapEntry<'a> = dashmap::mapref::entry::Entry<'a, Pubkey, PageRc>;
 
@@ -171,7 +177,9 @@ type AddressMapEntry<'a> = dashmap::mapref::entry::Entry<'a, Pubkey, PageRc>;
 #[derive(Default)]
 pub struct AddressBook {
     book: AddressMap,
-    newly_uncontended_addresses: AddressSet,
+    uncontended_task_ids: WeightedTaskIds,
+    runnable_guaranteed_task_ids: WeightedTaskIds,
+    guaranteed_lock_counts: std:collections::HashMap<UniqueWeight, usize>, 
 }
 
 impl AddressBook {
@@ -234,9 +242,18 @@ impl AddressBook {
         a.target.page().contended_unique_weights.remove(unique_weight);
     }
 
-    fn ensure_unlock(&mut self, attempt: &mut LockAttempt) {
-        if attempt.is_success() {
-            self.unlock(attempt);
+    fn reset_lock(&mut self, attempt: &mut LockAttempt) -> bool {
+        match attempt.status {
+            LockStatus::Succeded => {
+                self.unlock(attempt)
+            },
+            LockStatus::Guaranteed => {
+                self.cancel(attempt);
+                false
+            }
+            LockStatus::Failed => {
+                false // do nothing
+            }
         }
     }
 
@@ -271,6 +288,7 @@ impl AddressBook {
 
         if newly_uncontended {
             page.current_usage = Usage::Unused;
+            page.
             if !page.contended_unique_weights.is_empty() {
                 still_queued = true;
             }
@@ -470,9 +488,9 @@ impl ScheduleStage {
     */
 
     #[inline(never)]
-    fn get_weight_from_contended(address_book: &AddressBook) -> Option<UniqueWeight> {
-        trace!("n_u_a len(): {}", address_book.newly_uncontended_addresses.len());
-        address_book.newly_uncontended_addresses.last().map(|w| *w)
+    fn get_heaviest_from_contended(address_book: &AddressBook) -> Option<TaskId> {
+        trace!("n_u_a len(): {}", address_book.uncontended_task_ids.len());
+        address_book.uncontended_task_ids.last().map(|w| *w)
     }
 
     #[inline(never)]
@@ -486,7 +504,7 @@ impl ScheduleStage {
     )> {
         match (
             runnable_queue.heaviest_entry_to_execute(),
-            Self::get_weight_from_contended(address_book),
+            Self::get_heaviest_from_contended(address_book),
         ) {
             (Some(heaviest_runnable_entry), None) => {
                 trace!("select: runnable only");
@@ -575,12 +593,11 @@ impl ScheduleStage {
                 continue;
             } else if guaranteed_count > 0 {
                 assert!(!from_runnable);
-                address_book.guaranteed_locks.insert(unique_weight, guaranteed_count);
-                Self::ensure_unlock_for_guaranteed__execution(
+                Self::finalize_lock_for_guaranteed_execution(
                     address_book,
                     &unique_weight,
                     &mut populated_lock_attempts,
-                    from_runnable,
+                    guaranteed_count,
                 );
                 std::mem::swap(&mut next_task.tx.1, &mut populated_lock_attempts);
 
@@ -588,7 +605,7 @@ impl ScheduleStage {
             }
 
             trace!("successful lock: (from_runnable: {}) after {} contentions", from_runnable, next_task.contention_count);
-            Self::finalize_successful_lock_before_execution(
+            Self::finalize_lock_before_execution(
                 address_book,
                 &unique_weight,
                 &mut populated_lock_attempts,
@@ -605,7 +622,7 @@ impl ScheduleStage {
 
     #[inline(never)]
     // naming: relock_before_execution() / update_address_book() / update_uncontended_addresses()?
-    fn finalize_successful_lock_before_execution(
+    fn finalize_lock_before_execution(
         address_book: &mut AddressBook,
         unique_weight: &UniqueWeight,
         lock_attempts: &mut Vec<LockAttempt>,
@@ -615,8 +632,33 @@ impl ScheduleStage {
             address_book.forget_address_contention(&unique_weight, &mut l);
         }
 
-        // revert because now contended again
-        address_book.newly_uncontended_addresses.remove(&unique_weight);
+        address_book.uncontended_task_ids.remove(&unique_weight);
+    }
+
+    #[inline(never)]
+    fn finalize_lock_for_guaranteed_execution(
+        address_book: &mut AddressBook,
+        unique_weight: &UniqueWeight,
+        lock_attempts: &mut Vec<LockAttempt>,
+        guaranteed_count: usize,
+    ) {
+        for mut l in lock_attempts {
+            address_book.forget_address_contention(&unique_weight, &mut l);
+            match l.status {
+                LockStatus::Guaranteed => {
+                    l.status = LockStatus::Succeded;
+                    l.page().guaranteed_task_ids.insert(&unique_weight);
+                }
+                LockStatus::Succeded => {
+                    // do nothing
+                }
+                LockStatus::Failed => {
+                    unreachable!();
+                }
+            }
+        }
+        address_book.guaranteed_lock_counts.insert(unique_weight, guaranteed_count);
+        address_book.uncontended_task_ids.remove(&unique_weight);
     }
 
     #[inline(never)]
@@ -627,9 +669,9 @@ impl ScheduleStage {
         from_runnable: bool,
     ) {
         for l in lock_attempts {
-            address_book.ensure_unlock(l);
+            address_book.reset_lock(l);
             //if let Some(uw) = l.target.page().contended_unique_weights.last() {
-            //    address_book.newly_uncontended_addresses.remove(uw);
+            //    address_book.uncontended_task_ids.remove(uw);
             //}
 
             // todo: mem::forget and panic in LockAttempt::drop()
@@ -637,28 +679,36 @@ impl ScheduleStage {
 
         // revert because now contended again
         if !from_runnable {
-            //error!("n u a len() before: {}", address_book.newly_uncontended_addresses.len());
-            address_book.newly_uncontended_addresses.remove(&unique_weight);
-            //error!("n u a len() after: {}", address_book.newly_uncontended_addresses.len());
+            //error!("n u a len() before: {}", address_book.uncontended_task_ids.len());
+            address_book.uncontended_task_ids.remove(&unique_weight);
+            //error!("n u a len() after: {}", address_book.uncontended_task_ids.len());
         }
     }
 
     #[inline(never)]
     fn unlock_after_execution(address_book: &mut AddressBook, lock_attempts: Vec<LockAttempt>) {
         for mut l in lock_attempts.into_iter() {
-            let newly_uncontended_while_queued = address_book.unlock(&mut l);
-            if newly_uncontended_while_queued {
-                if let Some(uw) = l.target.page().contended_unique_weights.last() {
-                    address_book.newly_uncontended_addresses.insert(*uw);
+            let newly_uncontended_while_queued = address_book.reset_lock(&mut l);
+
+            let page = l.target.page();
+            if page.next_usage == Usage::Unused {
+                if newly_uncontended_while_queued {
+                    if let Some(uw) = page.contended_unique_weights.last() {
+                        address_book.uncontended_task_ids.insert(*uw);
+                    }
                 }
-                for uw in l.target.page().guaranteed_unique_weights {
-                    if let Some(count) = address_book.guaranteed_locks.get_mut(&uw) {
+            } else {
+                page.switch_to_next_usage();
+                for task_id in page.guaranteed_task_ids {
+                    if let Some(count) = address_book.guaranteed_lock_counts.get_mut(&uw) {
                         count -= 1;
                         if count == 0 {
-                            address_book.runnable_guaranteed_unique_weights.insert(uw);
+                            address_book.guaranteed_lock_counts.remove(&uw);
+                            address_book.runnable_guaranteed_task_ids.insert(uw);
                         }
                     }
                 }
+                page.guaranteed_task_ids.clear();
             }
 
             // todo: mem::forget and panic in LockAttempt::drop()
@@ -702,6 +752,12 @@ impl ScheduleStage {
         contended_queue: &mut TaskQueue,
         address_book: &mut AddressBook,
     ) -> Option<Box<ExecutionEnvironment>> {
+        if let Some(a) = address_book.runnable_guaranteed_task_ids.pop() {
+            let queue_entry = contended_queue.entry_to_execute(a)
+            let task = queue_entry.remove();
+            return Some(prepare_scheduled_execution(address_book, a, task, std::mem::take(&mut task.lock_attempts)));
+        }
+
         let maybe_ee =
             Self::pop_from_queue_then_lock(runnable_queue, contended_queue, address_book)
                 .map(|(uw, t, ll)| Self::prepare_scheduled_execution(address_book, uw, t, ll));
@@ -774,7 +830,7 @@ impl ScheduleStage {
             }
 
             loop {
-                /*if !address_book.newly_uncontended_addresses.is_empty() {
+                /*if !address_book.uncontended_task_ids.is_empty() {
                     trace!("prefer emptying n_u_a");
                 } else */ if executing_queue_count >= max_executing_queue_count {
                     trace!("outgoing queue full");
@@ -790,7 +846,7 @@ impl ScheduleStage {
 
                     to_execute_substage.send(ee).unwrap();
                 } else {
-                    trace!("incoming queue starved: n_u_a: {}", address_book.newly_uncontended_addresses.len());
+                    trace!("incoming queue starved: n_u_a: {}", address_book.uncontended_task_ids.len());
                     break;
                 }
             }

From 6cfb899c49ef2891bec1b1753223b20e18d93b3b Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Wed, 10 Aug 2022 10:35:37 +0900
Subject: [PATCH 0440/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index e3ad9c11b60592..2243b87e8d1704 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -179,7 +179,7 @@ pub struct AddressBook {
     book: AddressMap,
     uncontended_task_ids: WeightedTaskIds,
     runnable_guaranteed_task_ids: WeightedTaskIds,
-    guaranteed_lock_counts: std:collections::HashMap<UniqueWeight, usize>, 
+    guaranteed_lock_counts: std::collections::HashMap<UniqueWeight, usize>, 
 }
 
 impl AddressBook {

From 29e8f254815a92a9b10ac4832cc8a1f37c54e255 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Wed, 10 Aug 2022 10:37:33 +0900
Subject: [PATCH 0441/3199] save

---
 scheduler/src/lib.rs | 1 -
 1 file changed, 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 2243b87e8d1704..9539df20e0e41a 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -288,7 +288,6 @@ impl AddressBook {
 
         if newly_uncontended {
             page.current_usage = Usage::Unused;
-            page.
             if !page.contended_unique_weights.is_empty() {
                 still_queued = true;
             }

From d7e5dc0f3ffa3350e0dbbf1d2f0a61fb50e29a7f Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Wed, 10 Aug 2022 10:37:50 +0900
Subject: [PATCH 0442/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 9539df20e0e41a..2dd59a41c58844 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -752,7 +752,7 @@ impl ScheduleStage {
         address_book: &mut AddressBook,
     ) -> Option<Box<ExecutionEnvironment>> {
         if let Some(a) = address_book.runnable_guaranteed_task_ids.pop() {
-            let queue_entry = contended_queue.entry_to_execute(a)
+            let queue_entry = contended_queue.entry_to_execute(a);
             let task = queue_entry.remove();
             return Some(prepare_scheduled_execution(address_book, a, task, std::mem::take(&mut task.lock_attempts)));
         }

From 94bd4020556612c6fec334305ec43c4d83320959 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Wed, 10 Aug 2022 10:38:21 +0900
Subject: [PATCH 0443/3199] save

---
 scheduler/src/lib.rs | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 2dd59a41c58844..34b7d6332402d4 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -142,8 +142,8 @@ pub enum RequestedUsage {
 #[derive(Debug)]
 struct Page {
     current_usage: Usage,
-    contended_unique_weights: std::collections::BTreeSet<UniqueWeight>,
     next_usage: Usage,
+    contended_unique_weights: std::collections::BTreeSet<UniqueWeight>,
     guaranteed_task_ids: WeightedTaskIds,
     //loaded account from Accounts db
     //comulative_cu for qos; i.e. track serialized cumulative keyed by addresses and bail out block
@@ -156,6 +156,7 @@ impl Page {
             current_usage,
             next_usage: Usage::Unused,
             contended_unique_weights: Default::default(),
+            guaranteed_lock_counts: Default::default(),
         }
     }
 

From 8acf1e7f24d9a2d9cd37b863e6bb5b57c91e705d Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Wed, 10 Aug 2022 10:38:45 +0900
Subject: [PATCH 0444/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 34b7d6332402d4..83def8e1fb386f 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -156,7 +156,7 @@ impl Page {
             current_usage,
             next_usage: Usage::Unused,
             contended_unique_weights: Default::default(),
-            guaranteed_lock_counts: Default::default(),
+            guaranteed_task_ids: Default::default(),
         }
     }
 

From 6ad5a64baf44b9aedc28e4ac1ddb88704c49c63d Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Wed, 10 Aug 2022 10:39:08 +0900
Subject: [PATCH 0445/3199] save

---
 scheduler/src/lib.rs | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 83def8e1fb386f..101d0ff77c9363 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -700,11 +700,11 @@ impl ScheduleStage {
             } else {
                 page.switch_to_next_usage();
                 for task_id in page.guaranteed_task_ids {
-                    if let Some(count) = address_book.guaranteed_lock_counts.get_mut(&uw) {
+                    if let Some(count) = address_book.guaranteed_lock_counts.get_mut(&task_id) {
                         count -= 1;
                         if count == 0 {
-                            address_book.guaranteed_lock_counts.remove(&uw);
-                            address_book.runnable_guaranteed_task_ids.insert(uw);
+                            address_book.guaranteed_lock_counts.remove(&task_id);
+                            address_book.runnable_guaranteed_task_ids.insert(*task_id);
                         }
                     }
                 }

From 27ba93e50779957c6b34f17d53233416975f7149 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Wed, 10 Aug 2022 10:39:26 +0900
Subject: [PATCH 0446/3199] save

---
 scheduler/src/lib.rs | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 101d0ff77c9363..9b9d0e03c5cd99 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -249,7 +249,8 @@ impl AddressBook {
                 self.unlock(attempt)
             },
             LockStatus::Guaranteed => {
-                self.cancel(attempt);
+                panic!();
+                //self.cancel(attempt);
                 false
             }
             LockStatus::Failed => {

From 8aada3bbf59ae48800757f1720add641135a6144 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Wed, 10 Aug 2022 10:40:18 +0900
Subject: [PATCH 0447/3199] save

---
 scheduler/src/lib.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 9b9d0e03c5cd99..9c959dbfa72036 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -261,7 +261,7 @@ impl AddressBook {
 
     #[inline(never)]
     fn unlock(&mut self, attempt: &mut LockAttempt) -> bool {
-        debug_assert!(attempt.is_success());
+        //debug_assert!(attempt.is_success());
 
         let mut newly_uncontended = false;
         let mut still_queued = false;
@@ -406,7 +406,7 @@ fn attempt_lock_for_execution<'a>(
 
     for attempt in placeholder_attempts.iter_mut() {
         address_book.attempt_lock_address(from_runnable, unique_weight, attempt);
-        match address_book.status {
+        match attempt.status {
             LockStatus::Succeded => {},
             LockStatus::Failed => {
                 unlockable_count += 1;

From 6e00b272fd5a67539f6386a8e8d08079d8927494 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Wed, 10 Aug 2022 10:40:41 +0900
Subject: [PATCH 0448/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 9c959dbfa72036..12180f38df5d51 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -648,7 +648,7 @@ impl ScheduleStage {
             match l.status {
                 LockStatus::Guaranteed => {
                     l.status = LockStatus::Succeded;
-                    l.page().guaranteed_task_ids.insert(&unique_weight);
+                    l.target.page().guaranteed_task_ids.insert(&unique_weight);
                 }
                 LockStatus::Succeded => {
                     // do nothing

From 3435d7f62d33cdaed6a448153f7b314ad4f3a72a Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Wed, 10 Aug 2022 10:41:00 +0900
Subject: [PATCH 0449/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 12180f38df5d51..2aa467cbcc621c 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -648,7 +648,7 @@ impl ScheduleStage {
             match l.status {
                 LockStatus::Guaranteed => {
                     l.status = LockStatus::Succeded;
-                    l.target.page().guaranteed_task_ids.insert(&unique_weight);
+                    l.target.page().guaranteed_task_ids.insert(*unique_weight);
                 }
                 LockStatus::Succeded => {
                     // do nothing

From 672a116cffeb9758ecfe9a1229993404a9f5b732 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Wed, 10 Aug 2022 10:41:30 +0900
Subject: [PATCH 0450/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 2aa467cbcc621c..90aed04d6e9147 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -331,7 +331,7 @@ pub struct Weight {
                    // gas fee
 }
 
-#[derive(Copy, Clone, Debug, PartialEq, Eq, PartialOrd, Ord)]
+#[derive(Copy, Clone, Debug, Hash, PartialEq, Eq, PartialOrd, Ord)]
 pub struct UniqueWeight {
     // naming: Sequence Ordering?
     weight: Weight,

From 3a0349b05bd9134f8c91049a9a4ae2ca63b7205e Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Wed, 10 Aug 2022 10:41:48 +0900
Subject: [PATCH 0451/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 90aed04d6e9147..325ab60df53aeb 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -756,7 +756,7 @@ impl ScheduleStage {
         if let Some(a) = address_book.runnable_guaranteed_task_ids.pop() {
             let queue_entry = contended_queue.entry_to_execute(a);
             let task = queue_entry.remove();
-            return Some(prepare_scheduled_execution(address_book, a, task, std::mem::take(&mut task.lock_attempts)));
+            return Some(Self::prepare_scheduled_execution(address_book, a, task, std::mem::take(&mut task.lock_attempts)));
         }
 
         let maybe_ee =

From f3cb35ddf95e6a67e40a729d32dbe2b6f9d9bb53 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Wed, 10 Aug 2022 10:43:33 +0900
Subject: [PATCH 0452/3199] save

---
 scheduler/src/lib.rs | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 325ab60df53aeb..11e89d6a95007a 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -658,7 +658,7 @@ impl ScheduleStage {
                 }
             }
         }
-        address_book.guaranteed_lock_counts.insert(unique_weight, guaranteed_count);
+        address_book.guaranteed_lock_counts.insert(*unique_weight, guaranteed_count);
         address_book.uncontended_task_ids.remove(&unique_weight);
     }
 
@@ -702,10 +702,10 @@ impl ScheduleStage {
                 page.switch_to_next_usage();
                 for task_id in page.guaranteed_task_ids {
                     if let Some(count) = address_book.guaranteed_lock_counts.get_mut(&task_id) {
-                        count -= 1;
-                        if count == 0 {
+                        *count -= 1;
+                        if *count == 0 {
                             address_book.guaranteed_lock_counts.remove(&task_id);
-                            address_book.runnable_guaranteed_task_ids.insert(*task_id);
+                            address_book.runnable_guaranteed_task_ids.insert(task_id);
                         }
                     }
                 }
@@ -753,10 +753,10 @@ impl ScheduleStage {
         contended_queue: &mut TaskQueue,
         address_book: &mut AddressBook,
     ) -> Option<Box<ExecutionEnvironment>> {
-        if let Some(a) = address_book.runnable_guaranteed_task_ids.pop() {
+        if let Some(a) = address_book.runnable_guaranteed_task_ids.pop_last() {
             let queue_entry = contended_queue.entry_to_execute(a);
             let task = queue_entry.remove();
-            return Some(Self::prepare_scheduled_execution(address_book, a, task, std::mem::take(&mut task.lock_attempts)));
+            return Some(Self::prepare_scheduled_execution(address_book, a, task, std::mem::take(&mut task.tx.1)));
         }
 
         let maybe_ee =

From 067139369516411db8dd84a4afc8200db8fc83a2 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Wed, 10 Aug 2022 10:44:02 +0900
Subject: [PATCH 0453/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 11e89d6a95007a..a10e9c8ca07a19 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -324,7 +324,7 @@ impl Preloader {
     }
 }
 
-#[derive(Copy, Clone, Debug, PartialEq, Eq, PartialOrd, Ord)]
+#[derive(Copy, Clone, Debug, Hash, PartialEq, Eq, PartialOrd, Ord)]
 pub struct Weight {
     // naming: Sequence Ordering?
     pub ix: u64, // index in ledger entry?

From 918768bd20b610f37b03623db5438c96edd6f5b1 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Wed, 10 Aug 2022 10:44:47 +0900
Subject: [PATCH 0454/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index a10e9c8ca07a19..d6df33dc4d2069 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -111,7 +111,7 @@ impl LockAttempt {
 type UsageCount = usize;
 const SOLE_USE_COUNT: UsageCount = 1;
 
-#[derive(Debug, PartialEq)]
+#[derive(Copy, Clone, Debug, PartialEq)]
 enum Usage {
     Unused,
     // weight to abort running tx?

From 68d4080a7384bb61d127d3f091372246882448bc Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Wed, 10 Aug 2022 10:45:47 +0900
Subject: [PATCH 0455/3199] save

---
 scheduler/src/lib.rs | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index d6df33dc4d2069..c4b9eda3c71ae8 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -700,7 +700,7 @@ impl ScheduleStage {
                 }
             } else {
                 page.switch_to_next_usage();
-                for task_id in page.guaranteed_task_ids {
+                for task_id in std::mem::take(&mut page.guaranteed_task_ids) {
                     if let Some(count) = address_book.guaranteed_lock_counts.get_mut(&task_id) {
                         *count -= 1;
                         if *count == 0 {
@@ -709,7 +709,6 @@ impl ScheduleStage {
                         }
                     }
                 }
-                page.guaranteed_task_ids.clear();
             }
 
             // todo: mem::forget and panic in LockAttempt::drop()

From 70b57453941e6b51f75d44a46ef858348e61cb64 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Wed, 10 Aug 2022 10:46:29 +0900
Subject: [PATCH 0456/3199] save

---
 scheduler/src/lib.rs | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index c4b9eda3c71ae8..39479867958eaa 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -754,8 +754,9 @@ impl ScheduleStage {
     ) -> Option<Box<ExecutionEnvironment>> {
         if let Some(a) = address_book.runnable_guaranteed_task_ids.pop_last() {
             let queue_entry = contended_queue.entry_to_execute(a);
-            let task = queue_entry.remove();
-            return Some(Self::prepare_scheduled_execution(address_book, a, task, std::mem::take(&mut task.tx.1)));
+            let mut task = queue_entry.remove();
+            let ll = std::mem::take(&mut task.tx.1);
+            return Some(Self::prepare_scheduled_execution(address_book, a, task, ll));
         }
 
         let maybe_ee =

From 554d5ea7072817cb7e51a3ed3f43980f2246b01d Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Wed, 10 Aug 2022 10:55:56 +0900
Subject: [PATCH 0457/3199] save

---
 scheduler/src/lib.rs | 34 ++++++++++++++++++++++------------
 1 file changed, 22 insertions(+), 12 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 39479867958eaa..bb158c6391339f 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -222,12 +222,20 @@ impl AddressBook {
                             }
                         }
                     },
-                    Usage::Writable => match requested_usage {
-                        RequestedUsage::Readonly | RequestedUsage::Writable => {
-                            if from_runnable {
-                                Self::remember_address_contention(&mut page, unique_weight);
-                            }
+                    Usage::Writable => {
+                        if from_runnable {
+                            Self::remember_address_contention(&mut page, unique_weight);
                             *status = LockStatus::Failed;
+                        } else {
+                            match page.next_usage {
+                                Usage::Unused => {
+                                    *status = LockStatus::Guaranteed;
+                                    page.next_usage = Usage::renew(*requested_usage);
+                                },
+                                Usage::Readonly(_) | Usage::Writable => {
+                                    *status = LockStatus::Failed;
+                                },
+                            }
                         }
                     },
                 }
@@ -699,13 +707,15 @@ impl ScheduleStage {
                     }
                 }
             } else {
-                page.switch_to_next_usage();
-                for task_id in std::mem::take(&mut page.guaranteed_task_ids) {
-                    if let Some(count) = address_book.guaranteed_lock_counts.get_mut(&task_id) {
-                        *count -= 1;
-                        if *count == 0 {
-                            address_book.guaranteed_lock_counts.remove(&task_id);
-                            address_book.runnable_guaranteed_task_ids.insert(task_id);
+                if page.current_usage == Usage::Unused {
+                    page.switch_to_next_usage();
+                    for task_id in std::mem::take(&mut page.guaranteed_task_ids) {
+                        if let Some(count) = address_book.guaranteed_lock_counts.get_mut(&task_id) {
+                            *count -= 1;
+                            if *count == 0 {
+                                address_book.guaranteed_lock_counts.remove(&task_id);
+                                address_book.runnable_guaranteed_task_ids.insert(task_id);
+                            }
                         }
                     }
                 }

From 5aa7e80ecdc7017f02cc29026ed19920f42675e3 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Wed, 10 Aug 2022 10:58:17 +0900
Subject: [PATCH 0458/3199] save

---
 scheduler/src/lib.rs | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index bb158c6391339f..78b97d7cd4ac41 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -215,6 +215,7 @@ impl AddressBook {
                                         *status = LockStatus::Guaranteed;
                                         page.next_usage = Usage::renew(*requested_usage);
                                     },
+                                    // support multiple readonly locks!
                                     Usage::Readonly(_) | Usage::Writable => {
                                         *status = LockStatus::Failed;
                                     },
@@ -232,6 +233,7 @@ impl AddressBook {
                                     *status = LockStatus::Guaranteed;
                                     page.next_usage = Usage::renew(*requested_usage);
                                 },
+                                // support multiple readonly locks!
                                 Usage::Readonly(_) | Usage::Writable => {
                                     *status = LockStatus::Failed;
                                 },
@@ -702,6 +704,7 @@ impl ScheduleStage {
             let page = l.target.page();
             if page.next_usage == Usage::Unused {
                 if newly_uncontended_while_queued {
+                    // don't insert too eargarly for next_usage != Unused case
                     if let Some(uw) = page.contended_unique_weights.last() {
                         address_book.uncontended_task_ids.insert(*uw);
                     }

From 23f6a3ebc2204db50ab8cc2b11265021647d89ff Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Wed, 10 Aug 2022 14:15:48 +0900
Subject: [PATCH 0459/3199] save

---
 scheduler/src/lib.rs | 16 +++++++++-------
 1 file changed, 9 insertions(+), 7 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 78b97d7cd4ac41..b7ca294df0ccb5 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -559,6 +559,14 @@ impl ScheduleStage {
         contended_queue: &mut TaskQueue,
         address_book: &mut AddressBook,
     ) -> Option<(UniqueWeight, Task, Vec<LockAttempt>)> {
+        if let Some(a) = address_book.runnable_guaranteed_task_ids.pop_last() {
+            trace!("expediate pop from guaranteed queue");
+            let queue_entry = contended_queue.entry_to_execute(a);
+            let mut task = queue_entry.remove();
+            let ll = std::mem::take(&mut task.tx.1);
+            return Some(Self::prepare_scheduled_execution(address_book, a, task, ll));
+        }
+
         trace!("pop begin");
         loop {
         if let Some((reborrowed_contended_queue, mut queue_entry)) = Self::select_next_task(runnable_queue, contended_queue, address_book) {
@@ -604,6 +612,7 @@ impl ScheduleStage {
                 continue;
             } else if guaranteed_count > 0 {
                 assert!(!from_runnable);
+                trace!("guranteed exec: [{}/{}]", guaranteed_count, lock_count);
                 Self::finalize_lock_for_guaranteed_execution(
                     address_book,
                     &unique_weight,
@@ -765,13 +774,6 @@ impl ScheduleStage {
         contended_queue: &mut TaskQueue,
         address_book: &mut AddressBook,
     ) -> Option<Box<ExecutionEnvironment>> {
-        if let Some(a) = address_book.runnable_guaranteed_task_ids.pop_last() {
-            let queue_entry = contended_queue.entry_to_execute(a);
-            let mut task = queue_entry.remove();
-            let ll = std::mem::take(&mut task.tx.1);
-            return Some(Self::prepare_scheduled_execution(address_book, a, task, ll));
-        }
-
         let maybe_ee =
             Self::pop_from_queue_then_lock(runnable_queue, contended_queue, address_book)
                 .map(|(uw, t, ll)| Self::prepare_scheduled_execution(address_book, uw, t, ll));

From 27a9ff58aebc0ce1ee9594bb8022580a8d1f03d6 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Wed, 10 Aug 2022 14:16:30 +0900
Subject: [PATCH 0460/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index b7ca294df0ccb5..b239cf04deb274 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -564,7 +564,7 @@ impl ScheduleStage {
             let queue_entry = contended_queue.entry_to_execute(a);
             let mut task = queue_entry.remove();
             let ll = std::mem::take(&mut task.tx.1);
-            return Some(Self::prepare_scheduled_execution(address_book, a, task, ll));
+            return Some((unique_weight, task, ll));
         }
 
         trace!("pop begin");

From 0d684d527112ddda297e907df1a8ba87c47fd03b Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Wed, 10 Aug 2022 14:16:52 +0900
Subject: [PATCH 0461/3199] save

---
 scheduler/src/lib.rs | 1 +
 1 file changed, 1 insertion(+)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index b239cf04deb274..ca59893c8f6d04 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -612,6 +612,7 @@ impl ScheduleStage {
                 continue;
             } else if guaranteed_count > 0 {
                 assert!(!from_runnable);
+                let lock_count = populated_lock_attempts.len();
                 trace!("guranteed exec: [{}/{}]", guaranteed_count, lock_count);
                 Self::finalize_lock_for_guaranteed_execution(
                     address_book,

From b2b8443365c1eb29e7e7271e0f6872fd1517c917 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Wed, 10 Aug 2022 14:17:37 +0900
Subject: [PATCH 0462/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index ca59893c8f6d04..e397470844834c 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -564,7 +564,7 @@ impl ScheduleStage {
             let queue_entry = contended_queue.entry_to_execute(a);
             let mut task = queue_entry.remove();
             let ll = std::mem::take(&mut task.tx.1);
-            return Some((unique_weight, task, ll));
+            return Some((a, task, ll));
         }
 
         trace!("pop begin");

From fbca75a4e8980347dccefd315c29314137e44ba7 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Wed, 10 Aug 2022 15:58:13 +0900
Subject: [PATCH 0463/3199] save

---
 scheduler/src/lib.rs | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index e397470844834c..b055bef570a845 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -601,19 +601,20 @@ impl ScheduleStage {
                 next_task.contention_count += 1;
 
                 if from_runnable {
-                    trace!("move to contended due to lock failure [{}/{}]", unlockable_count, lock_count);
+                    trace!("move to contended due to lock failure [{}/{}/{}]", unlockable_count, guaranteed_count, lock_count);
                     reborrowed_contended_queue
                         .unwrap()
                         .add_to_schedule(*queue_entry.key(), queue_entry.remove());
                     // maybe run lightweight prune logic on contended_queue here.
                 } else {
-                    trace!("relock failed [{}/{}]; remains in contended: {:?} contention: {}", unlockable_count, lock_count, &unique_weight, next_task.contention_count);
+                    trace!("relock failed [{}/{}/{}]; remains in contended: {:?} contention: {}", unlockable_count, guaranteed_count, lock_count, &unique_weight, next_task.contention_count);
                 }
+
                 continue;
             } else if guaranteed_count > 0 {
                 assert!(!from_runnable);
                 let lock_count = populated_lock_attempts.len();
-                trace!("guranteed exec: [{}/{}]", guaranteed_count, lock_count);
+                trace!("guaranteed exec: [{}/{}]", guaranteed_count, lock_count);
                 Self::finalize_lock_for_guaranteed_execution(
                     address_book,
                     &unique_weight,

From d26f163e228f69b33f1a00d0305b5baffbe269b7 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Wed, 10 Aug 2022 16:02:41 +0900
Subject: [PATCH 0464/3199] save

---
 scheduler/src/lib.rs | 23 +++++++++++++++++++++--
 1 file changed, 21 insertions(+), 2 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index b055bef570a845..846ad7fe970d97 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -259,8 +259,7 @@ impl AddressBook {
                 self.unlock(attempt)
             },
             LockStatus::Guaranteed => {
-                panic!();
-                //self.cancel(attempt);
+                self.cancel(attempt);
                 false
             }
             LockStatus::Failed => {
@@ -308,6 +307,26 @@ impl AddressBook {
         still_queued
     }
 
+    #[inline(never)]
+    fn reset(&mut self, attempt: &mut LockAttempt) {
+        //debug_assert!(attempt.is_success());
+
+        let mut newly_uncontended = false;
+        let mut still_queued = false;
+
+        let mut page = attempt.target.page();
+
+        match page.next_usage {
+            Usage::Unused => {
+                unreachable!();
+            },
+            // support multiple readonly locks!
+            Usage::Readonly(_) | Usage::Writable => {
+                page.next_usage = Usage::Unused;
+            },
+        }
+    }
+
     pub fn preloader(&self) -> Preloader {
         Preloader{book: std::sync::Arc::clone(&self.book)}
     }

From fca305c0f8edbbcc36beebd74eec275d71645729 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Wed, 10 Aug 2022 16:03:02 +0900
Subject: [PATCH 0465/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 846ad7fe970d97..fa216de528a28c 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -308,7 +308,7 @@ impl AddressBook {
     }
 
     #[inline(never)]
-    fn reset(&mut self, attempt: &mut LockAttempt) {
+    fn cancel(&mut self, attempt: &mut LockAttempt) {
         //debug_assert!(attempt.is_success());
 
         let mut newly_uncontended = false;

From ca0be8f9a4e3574838f98277ac14e088354dd776 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Wed, 10 Aug 2022 16:03:39 +0900
Subject: [PATCH 0466/3199] save

---
 scheduler/src/lib.rs | 5 -----
 1 file changed, 5 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index fa216de528a28c..e2761b28feb923 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -309,11 +309,6 @@ impl AddressBook {
 
     #[inline(never)]
     fn cancel(&mut self, attempt: &mut LockAttempt) {
-        //debug_assert!(attempt.is_success());
-
-        let mut newly_uncontended = false;
-        let mut still_queued = false;
-
         let mut page = attempt.target.page();
 
         match page.next_usage {

From e6097af81a809105380905903f04dd28ff07de15 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Wed, 10 Aug 2022 16:09:15 +0900
Subject: [PATCH 0467/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index e2761b28feb923..1d38bea897a63d 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -574,7 +574,7 @@ impl ScheduleStage {
         address_book: &mut AddressBook,
     ) -> Option<(UniqueWeight, Task, Vec<LockAttempt>)> {
         if let Some(a) = address_book.runnable_guaranteed_task_ids.pop_last() {
-            trace!("expediate pop from guaranteed queue");
+            trace!("expediate pop from guaranteed queue [1/{}]", address_book.runnable_guaranteed_task_ids.len());
             let queue_entry = contended_queue.entry_to_execute(a);
             let mut task = queue_entry.remove();
             let ll = std::mem::take(&mut task.tx.1);

From 929eec19d09fd879524b20f8abe92c287455c7e9 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Wed, 10 Aug 2022 16:16:07 +0900
Subject: [PATCH 0468/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 1d38bea897a63d..d5ead71bf91c4f 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -574,7 +574,7 @@ impl ScheduleStage {
         address_book: &mut AddressBook,
     ) -> Option<(UniqueWeight, Task, Vec<LockAttempt>)> {
         if let Some(a) = address_book.runnable_guaranteed_task_ids.pop_last() {
-            trace!("expediate pop from guaranteed queue [1/{}]", address_book.runnable_guaranteed_task_ids.len());
+            trace!("expediate pop from guaranteed queue [rest: {}]", address_book.runnable_guaranteed_task_ids.len());
             let queue_entry = contended_queue.entry_to_execute(a);
             let mut task = queue_entry.remove();
             let ll = std::mem::take(&mut task.tx.1);

From c6d74736474bf98e5ace7ce5efb7762ea09e69ad Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Wed, 10 Aug 2022 16:24:40 +0900
Subject: [PATCH 0469/3199] save

---
 scheduler/src/lib.rs | 29 +++++++++++++----------------
 1 file changed, 13 insertions(+), 16 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index d5ead71bf91c4f..65ef601aa5e261 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -727,23 +727,20 @@ impl ScheduleStage {
             let newly_uncontended_while_queued = address_book.reset_lock(&mut l);
 
             let page = l.target.page();
-            if page.next_usage == Usage::Unused {
-                if newly_uncontended_while_queued {
-                    // don't insert too eargarly for next_usage != Unused case
-                    if let Some(uw) = page.contended_unique_weights.last() {
-                        address_book.uncontended_task_ids.insert(*uw);
-                    }
+            if newly_uncontended_while_queued {
+                // don't insert too eargarly for next_usage != Unused case
+                if let Some(uw) = page.contended_unique_weights.last() {
+                    address_book.uncontended_task_ids.insert(*uw);
                 }
-            } else {
-                if page.current_usage == Usage::Unused {
-                    page.switch_to_next_usage();
-                    for task_id in std::mem::take(&mut page.guaranteed_task_ids) {
-                        if let Some(count) = address_book.guaranteed_lock_counts.get_mut(&task_id) {
-                            *count -= 1;
-                            if *count == 0 {
-                                address_book.guaranteed_lock_counts.remove(&task_id);
-                                address_book.runnable_guaranteed_task_ids.insert(task_id);
-                            }
+            }
+            if page.current_usage == Usage::Unused {
+                page.switch_to_next_usage();
+                for task_id in std::mem::take(&mut page.guaranteed_task_ids) {
+                    if let Some(count) = address_book.guaranteed_lock_counts.get_mut(&task_id) {
+                        *count -= 1;
+                        if *count == 0 {
+                            address_book.guaranteed_lock_counts.remove(&task_id);
+                            address_book.runnable_guaranteed_task_ids.insert(task_id);
                         }
                     }
                 }

From add0a6376171d13a7f23d7522b769be78d0233f3 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Wed, 10 Aug 2022 16:29:42 +0900
Subject: [PATCH 0470/3199] save

---
 scheduler/src/lib.rs | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 65ef601aa5e261..9ddbbe7e28bb03 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -210,6 +210,7 @@ impl AddressBook {
                                 Self::remember_address_contention(&mut page, unique_weight);
                                 *status = LockStatus::Failed;
                             } else {
+                                /*
                                 match page.next_usage {
                                     Usage::Unused => {
                                         *status = LockStatus::Guaranteed;
@@ -220,6 +221,8 @@ impl AddressBook {
                                         *status = LockStatus::Failed;
                                     },
                                 }
+                                */
+                                *status = LockStatus::Failed;
                             }
                         }
                     },
@@ -228,6 +231,7 @@ impl AddressBook {
                             Self::remember_address_contention(&mut page, unique_weight);
                             *status = LockStatus::Failed;
                         } else {
+                            /*
                             match page.next_usage {
                                 Usage::Unused => {
                                     *status = LockStatus::Guaranteed;
@@ -238,6 +242,8 @@ impl AddressBook {
                                     *status = LockStatus::Failed;
                                 },
                             }
+                            */
+                            *status = LockStatus::Failed;
                         }
                     },
                 }

From 63b384b9fdc1fad8edcb6b75c07551c6a0062f42 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Wed, 10 Aug 2022 16:36:12 +0900
Subject: [PATCH 0471/3199] save

---
 scheduler/src/lib.rs | 12 +++++-------
 1 file changed, 5 insertions(+), 7 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 9ddbbe7e28bb03..f7fc4d9fcca82f 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -734,7 +734,6 @@ impl ScheduleStage {
 
             let page = l.target.page();
             if newly_uncontended_while_queued {
-                // don't insert too eargarly for next_usage != Unused case
                 if let Some(uw) = page.contended_unique_weights.last() {
                     address_book.uncontended_task_ids.insert(*uw);
                 }
@@ -742,12 +741,11 @@ impl ScheduleStage {
             if page.current_usage == Usage::Unused {
                 page.switch_to_next_usage();
                 for task_id in std::mem::take(&mut page.guaranteed_task_ids) {
-                    if let Some(count) = address_book.guaranteed_lock_counts.get_mut(&task_id) {
-                        *count -= 1;
-                        if *count == 0 {
-                            address_book.guaranteed_lock_counts.remove(&task_id);
-                            address_book.runnable_guaranteed_task_ids.insert(task_id);
-                        }
+                    let count = address_book.guaranteed_lock_counts.get_mut(&task_id).unwrap();
+                    *count -= 1;
+                    if *count == 0 {
+                        address_book.guaranteed_lock_counts.remove(&task_id);
+                        address_book.runnable_guaranteed_task_ids.insert(task_id);
                     }
                 }
             }

From c0759adedcc275893b2a1dde8ab88b58b4705423 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Wed, 10 Aug 2022 16:36:36 +0900
Subject: [PATCH 0472/3199] save

---
 scheduler/src/lib.rs | 9 +++------
 1 file changed, 3 insertions(+), 6 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index f7fc4d9fcca82f..6daf946ce99c58 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -210,7 +210,6 @@ impl AddressBook {
                                 Self::remember_address_contention(&mut page, unique_weight);
                                 *status = LockStatus::Failed;
                             } else {
-                                /*
                                 match page.next_usage {
                                     Usage::Unused => {
                                         *status = LockStatus::Guaranteed;
@@ -221,8 +220,8 @@ impl AddressBook {
                                         *status = LockStatus::Failed;
                                     },
                                 }
-                                */
-                                *status = LockStatus::Failed;
+                                
+                                //*status = LockStatus::Failed;
                             }
                         }
                     },
@@ -231,7 +230,6 @@ impl AddressBook {
                             Self::remember_address_contention(&mut page, unique_weight);
                             *status = LockStatus::Failed;
                         } else {
-                            /*
                             match page.next_usage {
                                 Usage::Unused => {
                                     *status = LockStatus::Guaranteed;
@@ -242,8 +240,7 @@ impl AddressBook {
                                     *status = LockStatus::Failed;
                                 },
                             }
-                            */
-                            *status = LockStatus::Failed;
+                            //*status = LockStatus::Failed;
                         }
                     },
                 }

From fa1f39ddc092949c4a8b2cc27ed5983893cef352 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Wed, 10 Aug 2022 21:31:27 +0900
Subject: [PATCH 0473/3199] save

---
 scheduler/src/lib.rs | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 6daf946ce99c58..357054ce7c9262 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -697,6 +697,7 @@ impl ScheduleStage {
             }
         }
         address_book.guaranteed_lock_counts.insert(*unique_weight, guaranteed_count);
+        trace!("guaranteed_lock_counts: {}", address_book.guaranteed_lock_counts.len());
         address_book.uncontended_task_ids.remove(&unique_weight);
     }
 
@@ -739,6 +740,7 @@ impl ScheduleStage {
                 page.switch_to_next_usage();
                 for task_id in std::mem::take(&mut page.guaranteed_task_ids) {
                     let count = address_book.guaranteed_lock_counts.get_mut(&task_id).unwrap();
+                    trace!("guaranteed lock decrease: {} => {}", *count, *count -1);
                     *count -= 1;
                     if *count == 0 {
                         address_book.guaranteed_lock_counts.remove(&task_id);

From 0944353de1a2272d0979e731338aea32ed7cc689 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Wed, 10 Aug 2022 21:43:18 +0900
Subject: [PATCH 0474/3199] save

---
 scheduler/src/lib.rs | 17 +++++++++++++----
 1 file changed, 13 insertions(+), 4 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 357054ce7c9262..2bc4e3b3c2a672 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -197,13 +197,22 @@ impl AddressBook {
 
                 match page.current_usage {
                     Usage::Unused => {
+                        assert_eq!(page.next_usage, Usage::Unused);
                         page.current_usage = Usage::renew(*requested_usage);
                         *status = LockStatus::Succeded;
                     }
                     Usage::Readonly(ref mut count) => match requested_usage {
                         RequestedUsage::Readonly => {
-                            *count += 1;
-                            *status = LockStatus::Succeded;
+                            // prevent read-lock from runnable too
+                            match page.next_usage {
+                                Usage::Unused => {
+                                    *count += 1;
+                                    *status = LockStatus::Succeded;
+                                },
+                                Usage::Readonly(_) | Usage::Writable => {
+                                    *status = LockStatus::Failed;
+                                }
+                            }
                         }
                         RequestedUsage::Writable => {
                             if from_runnable {
@@ -731,12 +740,12 @@ impl ScheduleStage {
             let newly_uncontended_while_queued = address_book.reset_lock(&mut l);
 
             let page = l.target.page();
-            if newly_uncontended_while_queued {
+            if newly_uncontended_while_queued && page.next_usage == Usage::Unused {
                 if let Some(uw) = page.contended_unique_weights.last() {
                     address_book.uncontended_task_ids.insert(*uw);
                 }
             }
-            if page.current_usage == Usage::Unused {
+            if page.current_usage == Usage::Unused && page.next_usage != Usage::Unused {
                 page.switch_to_next_usage();
                 for task_id in std::mem::take(&mut page.guaranteed_task_ids) {
                     let count = address_book.guaranteed_lock_counts.get_mut(&task_id).unwrap();

From 7f49a0c2123685186409ec580199efc3d183dce9 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Wed, 10 Aug 2022 21:52:30 +0900
Subject: [PATCH 0475/3199] save

---
 scheduler/src/lib.rs | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 2bc4e3b3c2a672..bb46a06fdaef7b 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -162,7 +162,6 @@ impl Page {
 
     fn switch_to_next_usage(&mut self) {
         self.current_usage = self.next_usage;
-        self.next_usage = Usage::Unused;
     }
 }
 
@@ -272,7 +271,7 @@ impl AddressBook {
             },
             LockStatus::Guaranteed => {
                 self.cancel(attempt);
-                false
+                self.unlock(attempt)
             }
             LockStatus::Failed => {
                 false // do nothing
@@ -694,7 +693,6 @@ impl ScheduleStage {
             address_book.forget_address_contention(&unique_weight, &mut l);
             match l.status {
                 LockStatus::Guaranteed => {
-                    l.status = LockStatus::Succeded;
                     l.target.page().guaranteed_task_ids.insert(*unique_weight);
                 }
                 LockStatus::Succeded => {

From 351d7c97161eba294775e498280b22becc7f3a9d Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Wed, 10 Aug 2022 21:57:29 +0900
Subject: [PATCH 0476/3199] save

---
 scheduler/src/lib.rs | 12 ++++++++----
 1 file changed, 8 insertions(+), 4 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index bb46a06fdaef7b..45adbff19fe8c2 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -264,14 +264,18 @@ impl AddressBook {
         a.target.page().contended_unique_weights.remove(unique_weight);
     }
 
-    fn reset_lock(&mut self, attempt: &mut LockAttempt) -> bool {
+    fn reset_lock(&mut self, attempt: &mut LockAttempt, after_execution: bool) -> bool {
         match attempt.status {
             LockStatus::Succeded => {
                 self.unlock(attempt)
             },
             LockStatus::Guaranteed => {
                 self.cancel(attempt);
-                self.unlock(attempt)
+                if after_execution {
+                    self.unlock(attempt)
+                } else {
+                    false
+                }
             }
             LockStatus::Failed => {
                 false // do nothing
@@ -716,7 +720,7 @@ impl ScheduleStage {
         from_runnable: bool,
     ) {
         for l in lock_attempts {
-            address_book.reset_lock(l);
+            address_book.reset_lock(l, false);
             //if let Some(uw) = l.target.page().contended_unique_weights.last() {
             //    address_book.uncontended_task_ids.remove(uw);
             //}
@@ -735,7 +739,7 @@ impl ScheduleStage {
     #[inline(never)]
     fn unlock_after_execution(address_book: &mut AddressBook, lock_attempts: Vec<LockAttempt>) {
         for mut l in lock_attempts.into_iter() {
-            let newly_uncontended_while_queued = address_book.reset_lock(&mut l);
+            let newly_uncontended_while_queued = address_book.reset_lock(&mut l, true);
 
             let page = l.target.page();
             if newly_uncontended_while_queued && page.next_usage == Usage::Unused {

From 64b7f1437b5723b4931a12f421266ee02f710ddb Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Wed, 10 Aug 2022 22:33:04 +0900
Subject: [PATCH 0477/3199] save

---
 scheduler/src/lib.rs | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 45adbff19fe8c2..c8eb768db441a0 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -215,7 +215,7 @@ impl AddressBook {
                         }
                         RequestedUsage::Writable => {
                             if from_runnable {
-                                Self::remember_address_contention(&mut page, unique_weight);
+                                Self::remember_address_contention(&mut page, unique_weight, &mut attempt);
                                 *status = LockStatus::Failed;
                             } else {
                                 match page.next_usage {
@@ -235,7 +235,7 @@ impl AddressBook {
                     },
                     Usage::Writable => {
                         if from_runnable {
-                            Self::remember_address_contention(&mut page, unique_weight);
+                            Self::remember_address_contention(&mut page, unique_weight, &mut attempt);
                             *status = LockStatus::Failed;
                         } else {
                             match page.next_usage {
@@ -255,12 +255,12 @@ impl AddressBook {
     }
 
     #[inline(never)]
-    fn remember_address_contention(page: &mut Page, unique_weight: &UniqueWeight) {
-        page.contended_unique_weights.insert(*unique_weight);
+    fn remember_address_contention(unique_weight: &UniqueWeight, a: &mut LockAttempt) {
+        a.target.page().contended_unique_weights.insert(*unique_weight);
     }
 
     #[inline(never)]
-    fn forget_address_contention(&mut self, unique_weight: &UniqueWeight, a: &mut LockAttempt) {
+    fn forget_address_contention(unique_weight: &UniqueWeight, a: &mut LockAttempt) {
         a.target.page().contended_unique_weights.remove(unique_weight);
     }
 

From 6eb838ecba00ade91a93eddf9e836938382a9463 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Wed, 10 Aug 2022 22:33:50 +0900
Subject: [PATCH 0478/3199] save

---
 scheduler/src/lib.rs | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index c8eb768db441a0..6e963da552728a 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -215,7 +215,7 @@ impl AddressBook {
                         }
                         RequestedUsage::Writable => {
                             if from_runnable {
-                                Self::remember_address_contention(&mut page, unique_weight, &mut attempt);
+                                Self::remember_address_contention(unique_weight, &mut attempt);
                                 *status = LockStatus::Failed;
                             } else {
                                 match page.next_usage {
@@ -235,7 +235,7 @@ impl AddressBook {
                     },
                     Usage::Writable => {
                         if from_runnable {
-                            Self::remember_address_contention(&mut page, unique_weight, &mut attempt);
+                            Self::remember_address_contention(unique_weight, &mut attempt);
                             *status = LockStatus::Failed;
                         } else {
                             match page.next_usage {
@@ -680,7 +680,7 @@ impl ScheduleStage {
     ) {
         for mut l in lock_attempts {
             // ensure to remove remaining refs of this unique_weight
-            address_book.forget_address_contention(&unique_weight, &mut l);
+            AddressBook::forget_address_contention(&unique_weight, &mut l);
         }
 
         address_book.uncontended_task_ids.remove(&unique_weight);
@@ -694,7 +694,7 @@ impl ScheduleStage {
         guaranteed_count: usize,
     ) {
         for mut l in lock_attempts {
-            address_book.forget_address_contention(&unique_weight, &mut l);
+            AddressBook::forget_address_contention(&unique_weight, &mut l);
             match l.status {
                 LockStatus::Guaranteed => {
                     l.target.page().guaranteed_task_ids.insert(*unique_weight);

From b0b8bf1a1213018f112a9729c2c0a5fca12aa5b2 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Wed, 10 Aug 2022 22:34:16 +0900
Subject: [PATCH 0479/3199] save

---
 scheduler/src/lib.rs | 2 --
 1 file changed, 2 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 6e963da552728a..7316dbb72d3dfe 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -190,8 +190,6 @@ impl AddressBook {
         unique_weight: &UniqueWeight,
         attempt: &mut LockAttempt,
     ) {
-        let LockAttempt {target, requested_usage, status} = attempt;
-
                 let mut page = unsafe { MyRcInner::get_mut_unchecked(&mut target.0) };
 
                 match page.current_usage {

From 1f33638e995af77696f499ba735486a142bfa678 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Wed, 10 Aug 2022 22:35:08 +0900
Subject: [PATCH 0480/3199] save

---
 scheduler/src/lib.rs | 16 +++++++++-------
 1 file changed, 9 insertions(+), 7 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 7316dbb72d3dfe..45adbff19fe8c2 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -190,6 +190,8 @@ impl AddressBook {
         unique_weight: &UniqueWeight,
         attempt: &mut LockAttempt,
     ) {
+        let LockAttempt {target, requested_usage, status} = attempt;
+
                 let mut page = unsafe { MyRcInner::get_mut_unchecked(&mut target.0) };
 
                 match page.current_usage {
@@ -213,7 +215,7 @@ impl AddressBook {
                         }
                         RequestedUsage::Writable => {
                             if from_runnable {
-                                Self::remember_address_contention(unique_weight, &mut attempt);
+                                Self::remember_address_contention(&mut page, unique_weight);
                                 *status = LockStatus::Failed;
                             } else {
                                 match page.next_usage {
@@ -233,7 +235,7 @@ impl AddressBook {
                     },
                     Usage::Writable => {
                         if from_runnable {
-                            Self::remember_address_contention(unique_weight, &mut attempt);
+                            Self::remember_address_contention(&mut page, unique_weight);
                             *status = LockStatus::Failed;
                         } else {
                             match page.next_usage {
@@ -253,12 +255,12 @@ impl AddressBook {
     }
 
     #[inline(never)]
-    fn remember_address_contention(unique_weight: &UniqueWeight, a: &mut LockAttempt) {
-        a.target.page().contended_unique_weights.insert(*unique_weight);
+    fn remember_address_contention(page: &mut Page, unique_weight: &UniqueWeight) {
+        page.contended_unique_weights.insert(*unique_weight);
     }
 
     #[inline(never)]
-    fn forget_address_contention(unique_weight: &UniqueWeight, a: &mut LockAttempt) {
+    fn forget_address_contention(&mut self, unique_weight: &UniqueWeight, a: &mut LockAttempt) {
         a.target.page().contended_unique_weights.remove(unique_weight);
     }
 
@@ -678,7 +680,7 @@ impl ScheduleStage {
     ) {
         for mut l in lock_attempts {
             // ensure to remove remaining refs of this unique_weight
-            AddressBook::forget_address_contention(&unique_weight, &mut l);
+            address_book.forget_address_contention(&unique_weight, &mut l);
         }
 
         address_book.uncontended_task_ids.remove(&unique_weight);
@@ -692,7 +694,7 @@ impl ScheduleStage {
         guaranteed_count: usize,
     ) {
         for mut l in lock_attempts {
-            AddressBook::forget_address_contention(&unique_weight, &mut l);
+            address_book.forget_address_contention(&unique_weight, &mut l);
             match l.status {
                 LockStatus::Guaranteed => {
                     l.target.page().guaranteed_task_ids.insert(*unique_weight);

From 4894537c1f5733ec36b7ca708526cab8244a203a Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 11 Aug 2022 16:12:16 +0900
Subject: [PATCH 0481/3199] save

---
 scheduler/src/lib.rs | 20 +++++---------------
 1 file changed, 5 insertions(+), 15 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 45adbff19fe8c2..ebfe664465c491 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -528,9 +528,9 @@ impl ScheduleStage {
     */
 
     #[inline(never)]
-    fn get_heaviest_from_contended(address_book: &AddressBook) -> Option<TaskId> {
+    fn get_heaviest_from_contended(address_book: &mut AddressBook) -> Option<TaskId> {
         trace!("n_u_a len(): {}", address_book.uncontended_task_ids.len());
-        address_book.uncontended_task_ids.last().map(|w| *w)
+        address_book.uncontended_task_ids.pop_last().map(|w| *w)
     }
 
     #[inline(never)]
@@ -618,8 +618,8 @@ impl ScheduleStage {
             );
 
             if unlockable_count > 0 {
-                //trace!("ensure_unlock_for_failed_execution(): {:?} {}", (&unique_weight, from_runnable), next_task.tx.0.signature());
-                Self::ensure_unlock_for_failed_execution(
+                //trace!("reset_lock_for_failed_execution(): {:?} {}", (&unique_weight, from_runnable), next_task.tx.0.signature());
+                Self::reset_lock_for_failed_execution(
                     address_book,
                     &unique_weight,
                     &mut populated_lock_attempts,
@@ -682,8 +682,6 @@ impl ScheduleStage {
             // ensure to remove remaining refs of this unique_weight
             address_book.forget_address_contention(&unique_weight, &mut l);
         }
-
-        address_book.uncontended_task_ids.remove(&unique_weight);
     }
 
     #[inline(never)]
@@ -709,11 +707,10 @@ impl ScheduleStage {
         }
         address_book.guaranteed_lock_counts.insert(*unique_weight, guaranteed_count);
         trace!("guaranteed_lock_counts: {}", address_book.guaranteed_lock_counts.len());
-        address_book.uncontended_task_ids.remove(&unique_weight);
     }
 
     #[inline(never)]
-    fn ensure_unlock_for_failed_execution(
+    fn reset_lock_for_failed_execution(
         address_book: &mut AddressBook,
         unique_weight: &UniqueWeight,
         lock_attempts: &mut Vec<LockAttempt>,
@@ -727,13 +724,6 @@ impl ScheduleStage {
 
             // todo: mem::forget and panic in LockAttempt::drop()
         }
-
-        // revert because now contended again
-        if !from_runnable {
-            //error!("n u a len() before: {}", address_book.uncontended_task_ids.len());
-            address_book.uncontended_task_ids.remove(&unique_weight);
-            //error!("n u a len() after: {}", address_book.uncontended_task_ids.len());
-        }
     }
 
     #[inline(never)]

From fb7a7ae6cfd80e575f099174123cbdaba35dcf0c Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 11 Aug 2022 16:12:32 +0900
Subject: [PATCH 0482/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index ebfe664465c491..87ab431be2d3ef 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -530,7 +530,7 @@ impl ScheduleStage {
     #[inline(never)]
     fn get_heaviest_from_contended(address_book: &mut AddressBook) -> Option<TaskId> {
         trace!("n_u_a len(): {}", address_book.uncontended_task_ids.len());
-        address_book.uncontended_task_ids.pop_last().map(|w| *w)
+        address_book.uncontended_task_ids.pop_last()
     }
 
     #[inline(never)]

From 17c8c22537efec0071f8e8649443a37f71081b4e Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 11 Aug 2022 16:13:21 +0900
Subject: [PATCH 0483/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 87ab431be2d3ef..e8ea05cbae5f6c 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -530,7 +530,7 @@ impl ScheduleStage {
     #[inline(never)]
     fn get_heaviest_from_contended(address_book: &mut AddressBook) -> Option<TaskId> {
         trace!("n_u_a len(): {}", address_book.uncontended_task_ids.len());
-        address_book.uncontended_task_ids.pop_last()
+        address_book.uncontended_task_ids.last_entry()
     }
 
     #[inline(never)]

From d40f8830d032551f010974eab72fb48bc94755b6 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 11 Aug 2022 16:13:55 +0900
Subject: [PATCH 0484/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index e8ea05cbae5f6c..b939890f601e90 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -169,7 +169,7 @@ impl Page {
 type AddressMap = std::sync::Arc<dashmap::DashMap<Pubkey, PageRc>>;
 use by_address::ByAddress;
 type TaskId = UniqueWeight;
-type WeightedTaskIds = std::collections::BTreeSet<TaskId>;
+type WeightedTaskIds = std::collections::BTreeMap<TaskId, ()>;
 //type AddressMapEntry<'a, K, V> = std::collections::hash_map::Entry<'a, K, V>;
 type AddressMapEntry<'a> = dashmap::mapref::entry::Entry<'a, Pubkey, PageRc>;
 

From 8c7bf46c21bfab8b7bf7a45ecf83a83bdac35253 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 11 Aug 2022 16:14:08 +0900
Subject: [PATCH 0485/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index b939890f601e90..a1a11033ac9bfb 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -745,7 +745,7 @@ impl ScheduleStage {
                     *count -= 1;
                     if *count == 0 {
                         address_book.guaranteed_lock_counts.remove(&task_id);
-                        address_book.runnable_guaranteed_task_ids.insert(task_id);
+                        address_book.runnable_guaranteed_task_ids.insert(task_id, ());
                     }
                 }
             }

From 25da8a3e030ad79902650d07e5d8c9b573bd3ff6 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 11 Aug 2022 21:43:39 +0900
Subject: [PATCH 0486/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index a1a11033ac9bfb..d7ff25b407bf7c 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -528,7 +528,7 @@ impl ScheduleStage {
     */
 
     #[inline(never)]
-    fn get_heaviest_from_contended(address_book: &mut AddressBook) -> Option<TaskId> {
+    fn get_heaviest_from_contended(address_book: &mut AddressBook) -> Option<std::collections::btree_map::OccupiedEntry<'a, UniqueWeight, ()>> {
         trace!("n_u_a len(): {}", address_book.uncontended_task_ids.len());
         address_book.uncontended_task_ids.last_entry()
     }

From 3e6d31d7d9c16092a1f8d04b94e003517c9ae493 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 11 Aug 2022 21:43:58 +0900
Subject: [PATCH 0487/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index d7ff25b407bf7c..2a687eee9b6a7e 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -528,7 +528,7 @@ impl ScheduleStage {
     */
 
     #[inline(never)]
-    fn get_heaviest_from_contended(address_book: &mut AddressBook) -> Option<std::collections::btree_map::OccupiedEntry<'a, UniqueWeight, ()>> {
+    fn get_heaviest_from_contended<'a>(address_book: &'a mut AddressBook) -> Option<std::collections::btree_map::OccupiedEntry<'a, UniqueWeight, ()>> {
         trace!("n_u_a len(): {}", address_book.uncontended_task_ids.len());
         address_book.uncontended_task_ids.last_entry()
     }

From 117187bf9e46f02ecb9e2a5b7843530c61ad18b2 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 11 Aug 2022 21:44:32 +0900
Subject: [PATCH 0488/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 2a687eee9b6a7e..0f03401d73efdc 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -554,7 +554,7 @@ impl ScheduleStage {
                 trace!("select: contended only");
                 Some((
                 None,
-                contended_queue.entry_to_execute(weight_from_contended),
+                contended_queue.entry_to_execute(weight_from_contended.remove()),
             ))
             },
             (Some(heaviest_runnable_entry), Some(weight_from_contended)) => {

From 1a14ae53cbca03c9812242eb8b3445049d1532bf Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 11 Aug 2022 21:45:26 +0900
Subject: [PATCH 0489/3199] save

---
 scheduler/src/lib.rs | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 0f03401d73efdc..96125e48b20423 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -552,9 +552,10 @@ impl ScheduleStage {
             }
             (None, Some(weight_from_contended)) => {
                 trace!("select: contended only");
-                Some((
-                None,
-                contended_queue.entry_to_execute(weight_from_contended.remove()),
+                let uw = weight_from_contended.key();
+                weight_from_contended.remove();
+
+                Some(( None, contended_queue.entry_to_execute(), uw))
             ))
             },
             (Some(heaviest_runnable_entry), Some(weight_from_contended)) => {

From 88adf120c35d5ed5f90aea13bb0a105907af7872 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 11 Aug 2022 21:45:38 +0900
Subject: [PATCH 0490/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 96125e48b20423..713fa9262a35c0 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -555,7 +555,7 @@ impl ScheduleStage {
                 let uw = weight_from_contended.key();
                 weight_from_contended.remove();
 
-                Some(( None, contended_queue.entry_to_execute(), uw))
+                Some(( None, contended_queue.entry_to_execute(), uw
             ))
             },
             (Some(heaviest_runnable_entry), Some(weight_from_contended)) => {

From 6fd1f26470c35dbf58286e449fb405a0434d91a4 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 11 Aug 2022 21:46:00 +0900
Subject: [PATCH 0491/3199] save

---
 scheduler/src/lib.rs | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 713fa9262a35c0..7f679be9d16561 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -555,8 +555,7 @@ impl ScheduleStage {
                 let uw = weight_from_contended.key();
                 weight_from_contended.remove();
 
-                Some(( None, contended_queue.entry_to_execute(), uw
-            ))
+                Some(( None, contended_queue.entry_to_execute(uw)))
             },
             (Some(heaviest_runnable_entry), Some(weight_from_contended)) => {
                 let weight_from_runnable = heaviest_runnable_entry.key();

From d232bc102755be3e1ce4509d3ffe65bf9e0294ae Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 11 Aug 2022 21:46:13 +0900
Subject: [PATCH 0492/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 7f679be9d16561..a0f56228554e9b 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -555,7 +555,7 @@ impl ScheduleStage {
                 let uw = weight_from_contended.key();
                 weight_from_contended.remove();
 
-                Some(( None, contended_queue.entry_to_execute(uw)))
+                Some(( None, contended_queue.entry_to_execute(*uw)))
             },
             (Some(heaviest_runnable_entry), Some(weight_from_contended)) => {
                 let weight_from_runnable = heaviest_runnable_entry.key();

From 8f4e3053f212eb10d17d4c46010f2b48f7529f36 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 11 Aug 2022 21:47:10 +0900
Subject: [PATCH 0493/3199] save

---
 scheduler/src/lib.rs | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index a0f56228554e9b..5e20c6f6c87e67 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -559,15 +559,17 @@ impl ScheduleStage {
             },
             (Some(heaviest_runnable_entry), Some(weight_from_contended)) => {
                 let weight_from_runnable = heaviest_runnable_entry.key();
+                let uw = weight_from_contended.key();
 
-                if weight_from_runnable > &weight_from_contended {
+                if weight_from_runnable > &uw {
                     trace!("select: runnable > contended");
                     Some((Some(contended_queue), heaviest_runnable_entry))
-                } else if &weight_from_contended > weight_from_runnable {
+                } else if &uw > weight_from_runnable {
                     trace!("select: contended > runnnable");
+                    weight_from_contended.remove();
                     Some((
                         None,
-                        contended_queue.entry_to_execute(weight_from_contended),
+                        contended_queue.entry_to_execute(uw),
                     ))
                 } else {
                     unreachable!(

From 10c83e7ec17df529a2cc246275dd757c5246b68f Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 11 Aug 2022 21:47:37 +0900
Subject: [PATCH 0494/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 5e20c6f6c87e67..67f964061e3f0c 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -564,7 +564,7 @@ impl ScheduleStage {
                 if weight_from_runnable > &uw {
                     trace!("select: runnable > contended");
                     Some((Some(contended_queue), heaviest_runnable_entry))
-                } else if &uw > weight_from_runnable {
+                } else if uw > weight_from_runnable {
                     trace!("select: contended > runnnable");
                     weight_from_contended.remove();
                     Some((

From 6894186e54820d5259350e1a52f15786709dbe39 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 11 Aug 2022 21:48:19 +0900
Subject: [PATCH 0495/3199] save

---
 scheduler/src/lib.rs | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 67f964061e3f0c..94bbc7a6321f22 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -569,7 +569,7 @@ impl ScheduleStage {
                     weight_from_contended.remove();
                     Some((
                         None,
-                        contended_queue.entry_to_execute(uw),
+                        contended_queue.entry_to_execute(*uw),
                     ))
                 } else {
                     unreachable!(
@@ -592,10 +592,10 @@ impl ScheduleStage {
     ) -> Option<(UniqueWeight, Task, Vec<LockAttempt>)> {
         if let Some(a) = address_book.runnable_guaranteed_task_ids.pop_last() {
             trace!("expediate pop from guaranteed queue [rest: {}]", address_book.runnable_guaranteed_task_ids.len());
-            let queue_entry = contended_queue.entry_to_execute(a);
+            let queue_entry = contended_queue.entry_to_execute(a.0);
             let mut task = queue_entry.remove();
             let ll = std::mem::take(&mut task.tx.1);
-            return Some((a, task, ll));
+            return Some((a.0, task, ll));
         }
 
         trace!("pop begin");

From 0856e0ca08171e393a7464a0d191583cc7dbbc7c Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 11 Aug 2022 21:49:08 +0900
Subject: [PATCH 0496/3199] save

---
 scheduler/src/lib.rs | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 94bbc7a6321f22..23c3065c83a891 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -697,7 +697,7 @@ impl ScheduleStage {
             address_book.forget_address_contention(&unique_weight, &mut l);
             match l.status {
                 LockStatus::Guaranteed => {
-                    l.target.page().guaranteed_task_ids.insert(*unique_weight);
+                    l.target.page().guaranteed_task_ids.insert(*unique_weight, ());
                 }
                 LockStatus::Succeded => {
                     // do nothing
@@ -736,12 +736,12 @@ impl ScheduleStage {
             let page = l.target.page();
             if newly_uncontended_while_queued && page.next_usage == Usage::Unused {
                 if let Some(uw) = page.contended_unique_weights.last() {
-                    address_book.uncontended_task_ids.insert(*uw);
+                    address_book.uncontended_task_ids.insert(*uw, ());
                 }
             }
             if page.current_usage == Usage::Unused && page.next_usage != Usage::Unused {
                 page.switch_to_next_usage();
-                for task_id in std::mem::take(&mut page.guaranteed_task_ids) {
+                for task_id in std::mem::take(&mut page.guaranteed_task_ids).keys() {
                     let count = address_book.guaranteed_lock_counts.get_mut(&task_id).unwrap();
                     trace!("guaranteed lock decrease: {} => {}", *count, *count -1);
                     *count -= 1;

From 5f58f0605bc0387e866837d452e599c3dabb954f Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 11 Aug 2022 21:49:23 +0900
Subject: [PATCH 0497/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 23c3065c83a891..0925825c3e080a 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -747,7 +747,7 @@ impl ScheduleStage {
                     *count -= 1;
                     if *count == 0 {
                         address_book.guaranteed_lock_counts.remove(&task_id);
-                        address_book.runnable_guaranteed_task_ids.insert(task_id, ());
+                        address_book.runnable_guaranteed_task_ids.insert(*task_id, ());
                     }
                 }
             }

From 11f932aff2af34315966673c4d3a33a05544638f Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 11 Aug 2022 21:49:42 +0900
Subject: [PATCH 0498/3199] save

---
 scheduler/src/lib.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 0925825c3e080a..56b49771e64e3d 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -552,10 +552,10 @@ impl ScheduleStage {
             }
             (None, Some(weight_from_contended)) => {
                 trace!("select: contended only");
-                let uw = weight_from_contended.key();
+                let uw = *weight_from_contended.key();
                 weight_from_contended.remove();
 
-                Some(( None, contended_queue.entry_to_execute(*uw)))
+                Some(( None, contended_queue.entry_to_execute(uw)))
             },
             (Some(heaviest_runnable_entry), Some(weight_from_contended)) => {
                 let weight_from_runnable = heaviest_runnable_entry.key();

From 98822b20d0ab75f4d429a2bb8b9ea069b43db5ef Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 11 Aug 2022 21:50:07 +0900
Subject: [PATCH 0499/3199] save

---
 scheduler/src/lib.rs | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 56b49771e64e3d..8f39bb24f446d8 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -559,9 +559,9 @@ impl ScheduleStage {
             },
             (Some(heaviest_runnable_entry), Some(weight_from_contended)) => {
                 let weight_from_runnable = heaviest_runnable_entry.key();
-                let uw = weight_from_contended.key();
+                let uw = *weight_from_contended.key();
 
-                if weight_from_runnable > &uw {
+                if weight_from_runnable > uw {
                     trace!("select: runnable > contended");
                     Some((Some(contended_queue), heaviest_runnable_entry))
                 } else if uw > weight_from_runnable {
@@ -569,7 +569,7 @@ impl ScheduleStage {
                     weight_from_contended.remove();
                     Some((
                         None,
-                        contended_queue.entry_to_execute(*uw),
+                        contended_queue.entry_to_execute(uw),
                     ))
                 } else {
                     unreachable!(

From b4f8b45fb9092ba6d6353fa934372dfed2c62393 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 11 Aug 2022 21:50:20 +0900
Subject: [PATCH 0500/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 8f39bb24f446d8..3fab9b309dd708 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -561,7 +561,7 @@ impl ScheduleStage {
                 let weight_from_runnable = heaviest_runnable_entry.key();
                 let uw = *weight_from_contended.key();
 
-                if weight_from_runnable > uw {
+                if weight_from_runnable > &uw {
                     trace!("select: runnable > contended");
                     Some((Some(contended_queue), heaviest_runnable_entry))
                 } else if uw > weight_from_runnable {

From fc791a75d4e780e46c125953e21d87428212251a Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 11 Aug 2022 21:50:33 +0900
Subject: [PATCH 0501/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 3fab9b309dd708..2273815600be5c 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -564,7 +564,7 @@ impl ScheduleStage {
                 if weight_from_runnable > &uw {
                     trace!("select: runnable > contended");
                     Some((Some(contended_queue), heaviest_runnable_entry))
-                } else if uw > weight_from_runnable {
+                } else if *uw > weight_from_runnable {
                     trace!("select: contended > runnnable");
                     weight_from_contended.remove();
                     Some((

From d20bb77d996d6825b1090cf6e1741de75e4d7ace Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 11 Aug 2022 21:50:42 +0900
Subject: [PATCH 0502/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 2273815600be5c..52070db26f81a3 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -564,7 +564,7 @@ impl ScheduleStage {
                 if weight_from_runnable > &uw {
                     trace!("select: runnable > contended");
                     Some((Some(contended_queue), heaviest_runnable_entry))
-                } else if *uw > weight_from_runnable {
+                } else if uw > *weight_from_runnable {
                     trace!("select: contended > runnnable");
                     weight_from_contended.remove();
                     Some((

From c70be0ebad7a26d9b3285ca6f23787cb902ec03b Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 11 Aug 2022 21:50:51 +0900
Subject: [PATCH 0503/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 52070db26f81a3..88f30964a6d2a9 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -564,7 +564,7 @@ impl ScheduleStage {
                 if weight_from_runnable > &uw {
                     trace!("select: runnable > contended");
                     Some((Some(contended_queue), heaviest_runnable_entry))
-                } else if uw > *weight_from_runnable {
+                } else if &uw > weight_from_runnable {
                     trace!("select: contended > runnnable");
                     weight_from_contended.remove();
                     Some((

From f2ba92b81f6762980564b56923a66a350bace814 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 11 Aug 2022 21:51:18 +0900
Subject: [PATCH 0504/3199] save

---
 scheduler/src/lib.rs | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 88f30964a6d2a9..c1c77e252d93a9 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -559,13 +559,14 @@ impl ScheduleStage {
             },
             (Some(heaviest_runnable_entry), Some(weight_from_contended)) => {
                 let weight_from_runnable = heaviest_runnable_entry.key();
-                let uw = *weight_from_contended.key();
+                let uw = weight_from_contended.key();
 
-                if weight_from_runnable > &uw {
+                if weight_from_runnable > uw {
                     trace!("select: runnable > contended");
                     Some((Some(contended_queue), heaviest_runnable_entry))
-                } else if &uw > weight_from_runnable {
+                } else if uw > weight_from_runnable {
                     trace!("select: contended > runnnable");
+                    let uw = *uw;
                     weight_from_contended.remove();
                     Some((
                         None,

From a13256879a71953508c1ac38a07c61a1d79d6622 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 11 Aug 2022 21:59:10 +0900
Subject: [PATCH 0505/3199] save

---
 scheduler/src/lib.rs | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index c1c77e252d93a9..829f3b6cb1109b 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -185,7 +185,6 @@ pub struct AddressBook {
 impl AddressBook {
     #[inline(never)]
     fn attempt_lock_address(
-        &mut self,
         from_runnable: bool,
         unique_weight: &UniqueWeight,
         attempt: &mut LockAttempt,
@@ -444,7 +443,7 @@ fn attempt_lock_for_execution<'a>(
     let mut guaranteed_count = 0;
 
     for attempt in placeholder_attempts.iter_mut() {
-        address_book.attempt_lock_address(from_runnable, unique_weight, attempt);
+        Self::attempt_lock_address(from_runnable, unique_weight, attempt);
         match attempt.status {
             LockStatus::Succeded => {},
             LockStatus::Failed => {

From f3cbc3aa78b7d1a035f6b6b12d83cc077b4a6adb Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 11 Aug 2022 21:59:28 +0900
Subject: [PATCH 0506/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 829f3b6cb1109b..2668711fea2bbd 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -443,7 +443,7 @@ fn attempt_lock_for_execution<'a>(
     let mut guaranteed_count = 0;
 
     for attempt in placeholder_attempts.iter_mut() {
-        Self::attempt_lock_address(from_runnable, unique_weight, attempt);
+        AddressBook::attempt_lock_address(from_runnable, unique_weight, attempt);
         match attempt.status {
             LockStatus::Succeded => {},
             LockStatus::Failed => {

From 252c1efdf4b59c9ad8d9511532054664de7903cb Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 11 Aug 2022 22:01:33 +0900
Subject: [PATCH 0507/3199] save

---
 scheduler/src/lib.rs                    | 2 +-
 transaction-scheduler-bench/src/main.rs | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 2668711fea2bbd..55555d68a67c26 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -442,7 +442,7 @@ fn attempt_lock_for_execution<'a>(
     let mut unlockable_count = 0;
     let mut guaranteed_count = 0;
 
-    for attempt in placeholder_attempts.iter_mut() {
+    for attempt in placeholder_attempts.par_iter_mut() {
         AddressBook::attempt_lock_address(from_runnable, unique_weight, attempt);
         match attempt.status {
             LockStatus::Succeded => {},
diff --git a/transaction-scheduler-bench/src/main.rs b/transaction-scheduler-bench/src/main.rs
index f1ad2602aee6e4..8db94f59350594 100644
--- a/transaction-scheduler-bench/src/main.rs
+++ b/transaction-scheduler-bench/src/main.rs
@@ -345,7 +345,7 @@ fn handle_transaction_batch(
             .unwrap();
 }
 
-const NUM_SENDERS: usize = 2;
+const NUM_SENDERS: usize = 1;
 
 fn spawn_packet_senders(
     preloader: Arc<solana_scheduler::Preloader>,

From 15c224353bfc891fb916ca573941b02c30bbac56 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 11 Aug 2022 22:01:46 +0900
Subject: [PATCH 0508/3199] save

---
 scheduler/src/lib.rs | 1 +
 1 file changed, 1 insertion(+)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 55555d68a67c26..c8bfaa728ce322 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -441,6 +441,7 @@ fn attempt_lock_for_execution<'a>(
     // no short-cuircuit; we at least all need to add to the contended queue
     let mut unlockable_count = 0;
     let mut guaranteed_count = 0;
+    use rayon::iter::IntoParallelRefMutIterator;
 
     for attempt in placeholder_attempts.par_iter_mut() {
         AddressBook::attempt_lock_address(from_runnable, unique_weight, attempt);

From 0927efbd9e92b51c740a76aeac4267ec3b2ee9e5 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 11 Aug 2022 22:02:28 +0900
Subject: [PATCH 0509/3199] save

---
 scheduler/Cargo.toml | 1 +
 scheduler/src/lib.rs | 1 -
 2 files changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/Cargo.toml b/scheduler/Cargo.toml
index bb159ba423148c..37337b89997cc2 100644
--- a/scheduler/Cargo.toml
+++ b/scheduler/Cargo.toml
@@ -15,6 +15,7 @@ atomic_enum = "0.1.1"
 by_address = "1.0.4"
 crossbeam-channel = "0.5.5"
 dashmap = { version = "4.0.2" }
+rayon = "1.5.3"
 log = "0.4.17"
 solana-entry = { path = "../entry", version = "=1.11.5" }
 solana-runtime = { path = "../runtime", version = "=1.11.5" }
diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index c8bfaa728ce322..55555d68a67c26 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -441,7 +441,6 @@ fn attempt_lock_for_execution<'a>(
     // no short-cuircuit; we at least all need to add to the contended queue
     let mut unlockable_count = 0;
     let mut guaranteed_count = 0;
-    use rayon::iter::IntoParallelRefMutIterator;
 
     for attempt in placeholder_attempts.par_iter_mut() {
         AddressBook::attempt_lock_address(from_runnable, unique_weight, attempt);

From 67db1ddd496b9cd97355a8f666a3add64da37dae Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 11 Aug 2022 22:02:38 +0900
Subject: [PATCH 0510/3199] save

---
 Cargo.lock           | 1 +
 scheduler/src/lib.rs | 1 +
 2 files changed, 2 insertions(+)

diff --git a/Cargo.lock b/Cargo.lock
index 1846dcf5313025..8f53bf829b134a 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -6078,6 +6078,7 @@ dependencies = [
  "dashmap",
  "log",
  "rand 0.7.3",
+ "rayon",
  "sha2 0.10.2",
  "solana-entry",
  "solana-logger 1.11.5",
diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 55555d68a67c26..c8bfaa728ce322 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -441,6 +441,7 @@ fn attempt_lock_for_execution<'a>(
     // no short-cuircuit; we at least all need to add to the contended queue
     let mut unlockable_count = 0;
     let mut guaranteed_count = 0;
+    use rayon::iter::IntoParallelRefMutIterator;
 
     for attempt in placeholder_attempts.par_iter_mut() {
         AddressBook::attempt_lock_address(from_runnable, unique_weight, attempt);

From 5a1628e95eab1cf75f645796dcfde28aa517cd3b Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 11 Aug 2022 22:03:14 +0900
Subject: [PATCH 0511/3199] save

---
 scheduler/src/lib.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index c8bfaa728ce322..0f0702ee835390 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -443,7 +443,7 @@ fn attempt_lock_for_execution<'a>(
     let mut guaranteed_count = 0;
     use rayon::iter::IntoParallelRefMutIterator;
 
-    for attempt in placeholder_attempts.par_iter_mut() {
+    placeholder_attempts.par_iter_mut().for_each(|attempt| {
         AddressBook::attempt_lock_address(from_runnable, unique_weight, attempt);
         match attempt.status {
             LockStatus::Succeded => {},
@@ -454,7 +454,7 @@ fn attempt_lock_for_execution<'a>(
                 guaranteed_count += 1;
             },
         }
-    }
+    });
 
     (unlockable_count, guaranteed_count, placeholder_attempts)
 }

From 5d1f298583e636e4794e4cba321644c50674121d Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 11 Aug 2022 22:03:30 +0900
Subject: [PATCH 0512/3199] save

---
 scheduler/src/lib.rs | 1 +
 1 file changed, 1 insertion(+)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 0f0702ee835390..d55bc261dff300 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -442,6 +442,7 @@ fn attempt_lock_for_execution<'a>(
     let mut unlockable_count = 0;
     let mut guaranteed_count = 0;
     use rayon::iter::IntoParallelRefMutIterator;
+    use rayon::iter::ParallelIterator;
 
     placeholder_attempts.par_iter_mut().for_each(|attempt| {
         AddressBook::attempt_lock_address(from_runnable, unique_weight, attempt);

From 92f671a1d2b09832d8d72d855b68f13c297b0548 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 11 Aug 2022 22:03:57 +0900
Subject: [PATCH 0513/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index d55bc261dff300..fd9a0c42c81137 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -439,7 +439,7 @@ fn attempt_lock_for_execution<'a>(
     mut placeholder_attempts: Vec<LockAttempt>,
 ) -> (usize, usize, Vec<LockAttempt>) {
     // no short-cuircuit; we at least all need to add to the contended queue
-    let mut unlockable_count = 0;
+    let mut unlockable_count = std::sync::atomic::AtomicUsize::new();
     let mut guaranteed_count = 0;
     use rayon::iter::IntoParallelRefMutIterator;
     use rayon::iter::ParallelIterator;

From 91dbbc5679dcff5071b96edbd9fbe264bd451c6f Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 11 Aug 2022 22:04:44 +0900
Subject: [PATCH 0514/3199] save

---
 scheduler/src/lib.rs | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index fd9a0c42c81137..0e11a9fd6bbc66 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -439,8 +439,8 @@ fn attempt_lock_for_execution<'a>(
     mut placeholder_attempts: Vec<LockAttempt>,
 ) -> (usize, usize, Vec<LockAttempt>) {
     // no short-cuircuit; we at least all need to add to the contended queue
-    let mut unlockable_count = std::sync::atomic::AtomicUsize::new();
-    let mut guaranteed_count = 0;
+    let mut unlockable_count = std::sync::atomic::AtomicUsize::new(0);
+    let mut guaranteed_count = std::sync::atomic::AtomicUsize::new(0);
     use rayon::iter::IntoParallelRefMutIterator;
     use rayon::iter::ParallelIterator;
 
@@ -449,10 +449,10 @@ fn attempt_lock_for_execution<'a>(
         match attempt.status {
             LockStatus::Succeded => {},
             LockStatus::Failed => {
-                unlockable_count += 1;
+                unlockable_count.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
             },
             LockStatus::Guaranteed => {
-                guaranteed_count += 1;
+                guaranteed_count.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
             },
         }
     });

From 8192b6d114ae8d2f026de35097f476939af58cf3 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 11 Aug 2022 22:04:56 +0900
Subject: [PATCH 0515/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 0e11a9fd6bbc66..fc1d32d4ab153c 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -457,7 +457,7 @@ fn attempt_lock_for_execution<'a>(
         }
     });
 
-    (unlockable_count, guaranteed_count, placeholder_attempts)
+    (unlockable_count.into(), guaranteed_count.into(), placeholder_attempts)
 }
 
 type PreprocessedTransaction = (SanitizedTransaction, Vec<LockAttempt>);

From b470dd90c34ef1a4d126f83a3e99904149d787de Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 11 Aug 2022 22:19:31 +0900
Subject: [PATCH 0516/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index fc1d32d4ab153c..765c7307e0f1ef 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -457,7 +457,7 @@ fn attempt_lock_for_execution<'a>(
         }
     });
 
-    (unlockable_count.into(), guaranteed_count.into(), placeholder_attempts)
+    (unlockable_count.load(std::sync::atomic::Ordering::Relaxed), guaranteed_count.load(std::sync::atomic::Ordering::Relaxed), placeholder_attempts)
 }
 
 type PreprocessedTransaction = (SanitizedTransaction, Vec<LockAttempt>);

From 5ab1be0a82430f0346d4d2d67de311e85d9b5f06 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 11 Aug 2022 22:21:25 +0900
Subject: [PATCH 0517/3199] save

---
 scheduler/src/lib.rs | 16 +++++++---------
 1 file changed, 7 insertions(+), 9 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 765c7307e0f1ef..2668711fea2bbd 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -439,25 +439,23 @@ fn attempt_lock_for_execution<'a>(
     mut placeholder_attempts: Vec<LockAttempt>,
 ) -> (usize, usize, Vec<LockAttempt>) {
     // no short-cuircuit; we at least all need to add to the contended queue
-    let mut unlockable_count = std::sync::atomic::AtomicUsize::new(0);
-    let mut guaranteed_count = std::sync::atomic::AtomicUsize::new(0);
-    use rayon::iter::IntoParallelRefMutIterator;
-    use rayon::iter::ParallelIterator;
+    let mut unlockable_count = 0;
+    let mut guaranteed_count = 0;
 
-    placeholder_attempts.par_iter_mut().for_each(|attempt| {
+    for attempt in placeholder_attempts.iter_mut() {
         AddressBook::attempt_lock_address(from_runnable, unique_weight, attempt);
         match attempt.status {
             LockStatus::Succeded => {},
             LockStatus::Failed => {
-                unlockable_count.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
+                unlockable_count += 1;
             },
             LockStatus::Guaranteed => {
-                guaranteed_count.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
+                guaranteed_count += 1;
             },
         }
-    });
+    }
 
-    (unlockable_count.load(std::sync::atomic::Ordering::Relaxed), guaranteed_count.load(std::sync::atomic::Ordering::Relaxed), placeholder_attempts)
+    (unlockable_count, guaranteed_count, placeholder_attempts)
 }
 
 type PreprocessedTransaction = (SanitizedTransaction, Vec<LockAttempt>);

From 2d12e5ea552465d4ce9b135273c2500b35eafc7f Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 11 Aug 2022 22:26:06 +0900
Subject: [PATCH 0518/3199] save

---
 scheduler/src/lib.rs | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 2668711fea2bbd..31f231e6dc8bd0 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -34,6 +34,7 @@ pub struct ExecutionEnvironment {
     //accounts: Vec<i8>,
     pub cu: usize,
     pub task: Task,
+    pub remembered: bool,
 }
 
 impl ExecutionEnvironment {
@@ -189,7 +190,7 @@ impl AddressBook {
         unique_weight: &UniqueWeight,
         attempt: &mut LockAttempt,
     ) {
-        let LockAttempt {target, requested_usage, status} = attempt;
+        let LockAttempt {target, requested_usage, status, remembered} = attempt;
 
                 let mut page = unsafe { MyRcInner::get_mut_unchecked(&mut target.0) };
 
@@ -215,6 +216,7 @@ impl AddressBook {
                         RequestedUsage::Writable => {
                             if from_runnable {
                                 Self::remember_address_contention(&mut page, unique_weight);
+                                *remembered = true;
                                 *status = LockStatus::Failed;
                             } else {
                                 match page.next_usage {
@@ -235,6 +237,7 @@ impl AddressBook {
                     Usage::Writable => {
                         if from_runnable {
                             Self::remember_address_contention(&mut page, unique_weight);
+                            *remembered = true;
                             *status = LockStatus::Failed;
                         } else {
                             match page.next_usage {

From b6066f48b835212341c00b31c57b32f939805f77 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 11 Aug 2022 22:26:30 +0900
Subject: [PATCH 0519/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 31f231e6dc8bd0..73e0a6692631d6 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -34,7 +34,6 @@ pub struct ExecutionEnvironment {
     //accounts: Vec<i8>,
     pub cu: usize,
     pub task: Task,
-    pub remembered: bool,
 }
 
 impl ExecutionEnvironment {
@@ -97,6 +96,7 @@ pub struct LockAttempt {
     target: PageRc,
     status: LockStatus,
     requested_usage: RequestedUsage,
+    remembered: bool,
 }
 
 impl LockAttempt {

From b6596c6d33b49b886d1a8b3b48f66f24bc84c47b Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 11 Aug 2022 22:26:40 +0900
Subject: [PATCH 0520/3199] save

---
 scheduler/src/lib.rs | 1 +
 1 file changed, 1 insertion(+)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 73e0a6692631d6..261330f309fa74 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -105,6 +105,7 @@ impl LockAttempt {
             target,
             status: LockStatus::Succeded,
             requested_usage,
+            remembered: false,
         }
     }
 }

From f49e1f682635a8db2c49bff25e8d711fc718bd69 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 11 Aug 2022 22:27:34 +0900
Subject: [PATCH 0521/3199] save

---
 scheduler/src/lib.rs | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 261330f309fa74..70f3a8c11757af 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -263,8 +263,10 @@ impl AddressBook {
     }
 
     #[inline(never)]
-    fn forget_address_contention(&mut self, unique_weight: &UniqueWeight, a: &mut LockAttempt) {
-        a.target.page().contended_unique_weights.remove(unique_weight);
+    fn forget_address_contention(unique_weight: &UniqueWeight, a: &mut LockAttempt) {
+        if a.remembered {
+            a.target.page().contended_unique_weights.remove(unique_weight);
+        }
     }
 
     fn reset_lock(&mut self, attempt: &mut LockAttempt, after_execution: bool) -> bool {
@@ -686,7 +688,7 @@ impl ScheduleStage {
     ) {
         for mut l in lock_attempts {
             // ensure to remove remaining refs of this unique_weight
-            address_book.forget_address_contention(&unique_weight, &mut l);
+            AddressBook::forget_address_contention(&unique_weight, &mut l);
         }
     }
 
@@ -698,7 +700,7 @@ impl ScheduleStage {
         guaranteed_count: usize,
     ) {
         for mut l in lock_attempts {
-            address_book.forget_address_contention(&unique_weight, &mut l);
+            AddressBook::forget_address_contention(&unique_weight, &mut l);
             match l.status {
                 LockStatus::Guaranteed => {
                     l.target.page().guaranteed_task_ids.insert(*unique_weight, ());

From 2de8e0d0c033502c705799b83cdfda695840286e Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 11 Aug 2022 22:32:48 +0900
Subject: [PATCH 0522/3199] save

---
 transaction-scheduler-bench/src/main.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/transaction-scheduler-bench/src/main.rs b/transaction-scheduler-bench/src/main.rs
index 8db94f59350594..f1ad2602aee6e4 100644
--- a/transaction-scheduler-bench/src/main.rs
+++ b/transaction-scheduler-bench/src/main.rs
@@ -345,7 +345,7 @@ fn handle_transaction_batch(
             .unwrap();
 }
 
-const NUM_SENDERS: usize = 1;
+const NUM_SENDERS: usize = 2;
 
 fn spawn_packet_senders(
     preloader: Arc<solana_scheduler::Preloader>,

From 00222d8843189bdbfc4f9703004705e3996b79cd Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 11 Aug 2022 22:42:12 +0900
Subject: [PATCH 0523/3199] save

---
 scheduler/src/lib.rs | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 70f3a8c11757af..42c0985d4ba5e4 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -748,7 +748,8 @@ impl ScheduleStage {
             if page.current_usage == Usage::Unused && page.next_usage != Usage::Unused {
                 page.switch_to_next_usage();
                 for task_id in std::mem::take(&mut page.guaranteed_task_ids).keys() {
-                    let count = address_book.guaranteed_lock_counts.get_mut(&task_id).unwrap();
+                    match address_book.guaranteed_lock_counts.entry(&task_id) {
+                    }
                     trace!("guaranteed lock decrease: {} => {}", *count, *count -1);
                     *count -= 1;
                     if *count == 0 {

From 1791f6c024a148bfd75bf6ecaf315136e2311c03 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 11 Aug 2022 22:42:45 +0900
Subject: [PATCH 0524/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 42c0985d4ba5e4..3daa8d2ae38190 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -748,7 +748,7 @@ impl ScheduleStage {
             if page.current_usage == Usage::Unused && page.next_usage != Usage::Unused {
                 page.switch_to_next_usage();
                 for task_id in std::mem::take(&mut page.guaranteed_task_ids).keys() {
-                    match address_book.guaranteed_lock_counts.entry(&task_id) {
+                    match address_book.guaranteed_lock_counts.entry(task_id) {
                     }
                     trace!("guaranteed lock decrease: {} => {}", *count, *count -1);
                     *count -= 1;

From 26f1944b86abbd736c2683e2572b077080bee1a8 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 11 Aug 2022 22:42:56 +0900
Subject: [PATCH 0525/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 3daa8d2ae38190..c923b79b130fe4 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -748,7 +748,7 @@ impl ScheduleStage {
             if page.current_usage == Usage::Unused && page.next_usage != Usage::Unused {
                 page.switch_to_next_usage();
                 for task_id in std::mem::take(&mut page.guaranteed_task_ids).keys() {
-                    match address_book.guaranteed_lock_counts.entry(task_id) {
+                    match address_book.guaranteed_lock_counts.entry(*task_id) {
                     }
                     trace!("guaranteed lock decrease: {} => {}", *count, *count -1);
                     *count -= 1;

From 902eb6691ed4aeed9fdc7de1ee2c10aa1f6077fa Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 11 Aug 2022 22:43:42 +0900
Subject: [PATCH 0526/3199] save

---
 scheduler/src/lib.rs | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index c923b79b130fe4..3701468c7131bc 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -749,6 +749,11 @@ impl ScheduleStage {
                 page.switch_to_next_usage();
                 for task_id in std::mem::take(&mut page.guaranteed_task_ids).keys() {
                     match address_book.guaranteed_lock_counts.entry(*task_id) {
+                        AddressMapEntry::Occupied(book_entry) => {
+                        }
+                        AddressMapEntry::Vacant(_) => {
+                            unreachable!();
+                        }
                     }
                     trace!("guaranteed lock decrease: {} => {}", *count, *count -1);
                     *count -= 1;

From 2db9c692cedcbe987436c8c84db07e4558c9d8e6 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 11 Aug 2022 22:44:33 +0900
Subject: [PATCH 0527/3199] save

---
 scheduler/src/lib.rs | 17 +++++++++--------
 1 file changed, 9 insertions(+), 8 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 3701468c7131bc..b0577967fa187a 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -749,18 +749,19 @@ impl ScheduleStage {
                 page.switch_to_next_usage();
                 for task_id in std::mem::take(&mut page.guaranteed_task_ids).keys() {
                     match address_book.guaranteed_lock_counts.entry(*task_id) {
-                        AddressMapEntry::Occupied(book_entry) => {
+                        std::collections::hash_map::Occupied(entry) => {
+                            let count = entry.get_mut();
+                            trace!("guaranteed lock decrease: {} => {}", *count, *count -1);
+                            *count -= 1;
+                            if *count == 0 {
+                                entry.remove();
+                                address_book.runnable_guaranteed_task_ids.insert(*task_id, ());
+                            }
                         }
-                        AddressMapEntry::Vacant(_) => {
+                        std::collections::hash_map::Vacant(_) => {
                             unreachable!();
                         }
                     }
-                    trace!("guaranteed lock decrease: {} => {}", *count, *count -1);
-                    *count -= 1;
-                    if *count == 0 {
-                        address_book.guaranteed_lock_counts.remove(&task_id);
-                        address_book.runnable_guaranteed_task_ids.insert(*task_id, ());
-                    }
                 }
             }
 

From 7b51dad8069f5acef381d7d3a2dc67939af17d46 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 11 Aug 2022 22:44:48 +0900
Subject: [PATCH 0528/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index b0577967fa187a..eab01c6a7a34c3 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -757,7 +757,7 @@ impl ScheduleStage {
                                 entry.remove();
                                 address_book.runnable_guaranteed_task_ids.insert(*task_id, ());
                             }
-                        }
+                        },
                         std::collections::hash_map::Vacant(_) => {
                             unreachable!();
                         }

From fcf273c463ce29ee193092015dd79eec53646844 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 11 Aug 2022 22:45:12 +0900
Subject: [PATCH 0529/3199] save

---
 scheduler/src/lib.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index eab01c6a7a34c3..49e37310875333 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -749,7 +749,7 @@ impl ScheduleStage {
                 page.switch_to_next_usage();
                 for task_id in std::mem::take(&mut page.guaranteed_task_ids).keys() {
                     match address_book.guaranteed_lock_counts.entry(*task_id) {
-                        std::collections::hash_map::Occupied(entry) => {
+                        std::collections::hash_map::Entry::Occupied(entry) => {
                             let count = entry.get_mut();
                             trace!("guaranteed lock decrease: {} => {}", *count, *count -1);
                             *count -= 1;
@@ -758,7 +758,7 @@ impl ScheduleStage {
                                 address_book.runnable_guaranteed_task_ids.insert(*task_id, ());
                             }
                         },
-                        std::collections::hash_map::Vacant(_) => {
+                        std::collections::hash_map::Entry::Vacant(_) => {
                             unreachable!();
                         }
                     }

From 84088222753f9b07f5cdbecb869aabbd51646b9e Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 11 Aug 2022 22:45:20 +0900
Subject: [PATCH 0530/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 49e37310875333..aaee49e56cd152 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -749,7 +749,7 @@ impl ScheduleStage {
                 page.switch_to_next_usage();
                 for task_id in std::mem::take(&mut page.guaranteed_task_ids).keys() {
                     match address_book.guaranteed_lock_counts.entry(*task_id) {
-                        std::collections::hash_map::Entry::Occupied(entry) => {
+                        std::collections::hash_map::Entry::Occupied(mut entry) => {
                             let count = entry.get_mut();
                             trace!("guaranteed lock decrease: {} => {}", *count, *count -1);
                             *count -= 1;

From 5f3b69f06f6904c93576f2c5636e323d7d590fe7 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 11 Aug 2022 22:46:55 +0900
Subject: [PATCH 0531/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index aaee49e56cd152..ee3df45c45c6e5 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -834,7 +834,7 @@ impl ScheduleStage {
         let mut current_unique_key = u64::max_value();
 
         loop {
-            trace!("schedule_once!");
+            trace!("schedule_once (runnnable: {}, contended: {}, guaranteed: {})!", runnable_queue.len(), contended_queue.len(), address_book.guaranteed_lock_counts.len());
 
             let i = from.recv().unwrap();
             match i {

From e7a7d3b55683e332e97c60b479d22cee8c894b4c Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 11 Aug 2022 22:47:35 +0900
Subject: [PATCH 0532/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index ee3df45c45c6e5..440496235998df 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -834,7 +834,7 @@ impl ScheduleStage {
         let mut current_unique_key = u64::max_value();
 
         loop {
-            trace!("schedule_once (runnnable: {}, contended: {}, guaranteed: {})!", runnable_queue.len(), contended_queue.len(), address_book.guaranteed_lock_counts.len());
+            trace!("schedule_once (runnnable: {}, contended: {}, guaranteed: {})!", runnable_queue.count(), contended_queue.count(), address_book.guaranteed_lock_counts.len());
 
             let i = from.recv().unwrap();
             match i {

From 144a2da57cc39e875a84c88d859e83248868b7c5 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 11 Aug 2022 22:47:49 +0900
Subject: [PATCH 0533/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 440496235998df..6927a7393d46e6 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -834,7 +834,7 @@ impl ScheduleStage {
         let mut current_unique_key = u64::max_value();
 
         loop {
-            trace!("schedule_once (runnnable: {}, contended: {}, guaranteed: {})!", runnable_queue.count(), contended_queue.count(), address_book.guaranteed_lock_counts.len());
+            trace!("schedule_once (runnnable: {}, contended: {}, guaranteed: {})!", runnable_queue.depth(), contended_queue.depth(), address_book.guaranteed_lock_counts.len());
 
             let i = from.recv().unwrap();
             match i {

From 06b12a71ba7bfe6f981029c1c446bb6f33d2c27f Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 11 Aug 2022 22:48:03 +0900
Subject: [PATCH 0534/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 6927a7393d46e6..8a02dce3b17142 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -834,7 +834,7 @@ impl ScheduleStage {
         let mut current_unique_key = u64::max_value();
 
         loop {
-            trace!("schedule_once (runnnable: {}, contended: {}, guaranteed: {})!", runnable_queue.depth(), contended_queue.depth(), address_book.guaranteed_lock_counts.len());
+            trace!("schedule_once (runnnable: {}, contended: {}, guaranteed: {})!", runnable_queue.task_count(), contended_queue.task_count(), address_book.guaranteed_lock_counts.len());
 
             let i = from.recv().unwrap();
             match i {

From 433ae818e8365c1c67fd9b500ca60412088bd7e5 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 11 Aug 2022 22:48:38 +0900
Subject: [PATCH 0535/3199] save

---
 scheduler/src/lib.rs | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 8a02dce3b17142..cfeff962e39c84 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -434,6 +434,10 @@ impl TaskQueue {
     ) -> Option<std::collections::btree_map::OccupiedEntry<'_, UniqueWeight, Task>> {
         self.tasks.last_entry()
     }
+
+    fn task_count(&self) -> usize {
+        self.tasks.len()
+    }
 }
 
 #[inline(never)]

From 6dce3ddc2314f7ee8f8a2553f441a92f5ee20b64 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 11 Aug 2022 22:52:00 +0900
Subject: [PATCH 0536/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index cfeff962e39c84..66f4764b52abbd 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -756,7 +756,7 @@ impl ScheduleStage {
                         std::collections::hash_map::Entry::Occupied(mut entry) => {
                             let count = entry.get_mut();
                             trace!("guaranteed lock decrease: {} => {}", *count, *count -1);
-                            *count -= 1;
+                            count.checked_sub(1).unwrap();
                             if *count == 0 {
                                 entry.remove();
                                 address_book.runnable_guaranteed_task_ids.insert(*task_id, ());

From bc4338329a09e80d2b5f739979a7af36a437dbb6 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 11 Aug 2022 22:54:44 +0900
Subject: [PATCH 0537/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 66f4764b52abbd..753ef9e4779eab 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -756,7 +756,7 @@ impl ScheduleStage {
                         std::collections::hash_map::Entry::Occupied(mut entry) => {
                             let count = entry.get_mut();
                             trace!("guaranteed lock decrease: {} => {}", *count, *count -1);
-                            count.checked_sub(1).unwrap();
+                            *count = count.checked_sub(1).unwrap();
                             if *count == 0 {
                                 entry.remove();
                                 address_book.runnable_guaranteed_task_ids.insert(*task_id, ());

From 921a513a0574abd2383c8907ab01058c7a550aa9 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 12 Aug 2022 09:47:07 +0900
Subject: [PATCH 0538/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 753ef9e4779eab..093243e5115ad6 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -838,7 +838,7 @@ impl ScheduleStage {
         let mut current_unique_key = u64::max_value();
 
         loop {
-            trace!("schedule_once (runnnable: {}, contended: {}, guaranteed: {})!", runnable_queue.task_count(), contended_queue.task_count(), address_book.guaranteed_lock_counts.len());
+            trace!("schedule_once (runnnable: {}, contended: {}, guaranteed: {}, exec: {})!", runnable_queue.task_count(), contended_queue.task_count(), address_book.guaranteed_lock_counts.len(), executing_queue_count);
 
             let i = from.recv().unwrap();
             match i {

From e17b2f042688cfa7bc21c359030931373190b773 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 12 Aug 2022 09:52:27 +0900
Subject: [PATCH 0539/3199] save

---
 scheduler/src/lib.rs | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 093243e5115ad6..3f796fa21d4dcb 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -203,15 +203,18 @@ impl AddressBook {
                     }
                     Usage::Readonly(ref mut count) => match requested_usage {
                         RequestedUsage::Readonly => {
-                            // prevent read-lock from runnable too
+                            // prevent newer read-locks (even from runnable too)
                             match page.next_usage {
                                 Usage::Unused => {
                                     *count += 1;
                                     *status = LockStatus::Succeded;
                                 },
-                                Usage::Readonly(_) | Usage::Writable => {
+                                Usage::Writable => {
                                     *status = LockStatus::Failed;
                                 }
+                                Usage::Readonly(_) => {
+                                    unreachable!();
+                                }
                             }
                         }
                         RequestedUsage::Writable => {

From 340685ab78529b0d7bdcd29fdbabd4f746dd1db1 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 12 Aug 2022 09:56:04 +0900
Subject: [PATCH 0540/3199] save

---
 scheduler/src/lib.rs | 15 +++++++--------
 1 file changed, 7 insertions(+), 8 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 3f796fa21d4dcb..81c4bbbb0e7c6e 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -209,18 +209,15 @@ impl AddressBook {
                                     *count += 1;
                                     *status = LockStatus::Succeded;
                                 },
-                                Usage::Writable => {
+                                Usage::Readonly(_) | Usage::Writable => {
                                     *status = LockStatus::Failed;
                                 }
-                                Usage::Readonly(_) => {
-                                    unreachable!();
-                                }
                             }
                         }
                         RequestedUsage::Writable => {
                             if from_runnable {
-                                Self::remember_address_contention(&mut page, unique_weight);
-                                *remembered = true;
+                                //Self::remember_address_contention(&mut page, unique_weight);
+                                //*remembered = true;
                                 *status = LockStatus::Failed;
                             } else {
                                 match page.next_usage {
@@ -240,8 +237,8 @@ impl AddressBook {
                     },
                     Usage::Writable => {
                         if from_runnable {
-                            Self::remember_address_contention(&mut page, unique_weight);
-                            *remembered = true;
+                            //Self::remember_address_contention(&mut page, unique_weight);
+                            //*remembered = true;
                             *status = LockStatus::Failed;
                         } else {
                             match page.next_usage {
@@ -732,6 +729,8 @@ impl ScheduleStage {
         from_runnable: bool,
     ) {
         for l in lock_attempts {
+            Self::remember_address_contention(&mut l.target.page(), unique_weight);
+            l.remembered = true;
             address_book.reset_lock(l, false);
             //if let Some(uw) = l.target.page().contended_unique_weights.last() {
             //    address_book.uncontended_task_ids.remove(uw);

From 9054edd94b9aed8717edbf82ac0d8b12d3dea2b5 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 12 Aug 2022 09:56:19 +0900
Subject: [PATCH 0541/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 81c4bbbb0e7c6e..df1b5aa3c56f4c 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -729,7 +729,7 @@ impl ScheduleStage {
         from_runnable: bool,
     ) {
         for l in lock_attempts {
-            Self::remember_address_contention(&mut l.target.page(), unique_weight);
+            AddressBook::remember_address_contention(&mut l.target.page(), unique_weight);
             l.remembered = true;
             address_book.reset_lock(l, false);
             //if let Some(uw) = l.target.page().contended_unique_weights.last() {

From a865b1269c3cf2d3efe33e4ae77044b46256347c Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 12 Aug 2022 09:59:17 +0900
Subject: [PATCH 0542/3199] save

---
 scheduler/src/lib.rs | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index df1b5aa3c56f4c..2727c692368c0b 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -729,8 +729,10 @@ impl ScheduleStage {
         from_runnable: bool,
     ) {
         for l in lock_attempts {
-            AddressBook::remember_address_contention(&mut l.target.page(), unique_weight);
-            l.remembered = true;
+            if from_runnable {
+                AddressBook::remember_address_contention(&mut l.target.page(), unique_weight);
+                l.remembered = true;
+            }
             address_book.reset_lock(l, false);
             //if let Some(uw) = l.target.page().contended_unique_weights.last() {
             //    address_book.uncontended_task_ids.remove(uw);

From eef23c60c0a4b5fcfe5730fd41ca78129df4c8c0 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 12 Aug 2022 10:06:58 +0900
Subject: [PATCH 0543/3199] save

---
 transaction-scheduler-bench/src/main.rs | 1 +
 1 file changed, 1 insertion(+)

diff --git a/transaction-scheduler-bench/src/main.rs b/transaction-scheduler-bench/src/main.rs
index f1ad2602aee6e4..b5895b0a2d9d27 100644
--- a/transaction-scheduler-bench/src/main.rs
+++ b/transaction-scheduler-bench/src/main.rs
@@ -513,6 +513,7 @@ fn build_packet(
         .iter()
         .map(|address| solana_scheduler::LockAttempt::new(preloader.load(**address), solana_scheduler::RequestedUsage::Readonly));
     let locks = writable_lock_iter.chain(readonly_lock_iter).collect::<Vec<_>>();
+    let p = solana_scheduler::get_transaction_priority_details(&sanitized_tx);
 
     Box::new((sanitized_tx, locks))
 }

From 37e1f6b359f7a4ca35a21652298f54f1a31b10d7 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 12 Aug 2022 10:07:15 +0900
Subject: [PATCH 0544/3199] save

---
 scheduler/src/lib.rs | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 2727c692368c0b..8d38e59e008a27 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -478,7 +478,7 @@ pub enum Multiplexed {
     FromExecute(Box<ExecutionEnvironment>),
 }
 
-fn get_transaction_priority_details(tx: &SanitizedTransaction) -> u64 {
+pub fn get_transaction_priority_details(tx: &SanitizedTransaction) -> u64 {
         use solana_program_runtime::compute_budget::ComputeBudget;
         let mut compute_budget = ComputeBudget::default();
         compute_budget
@@ -856,7 +856,6 @@ impl ScheduleStage {
 
                     for vv in vvv {
                         for v in vv {
-                            let p = get_transaction_priority_details(&v.0);
                             Self::register_runnable_task((Weight { ix: p }, v), runnable_queue, &mut current_unique_key);
                             if executing_queue_count < max_executing_queue_count {
                                 let maybe_ee =

From f76c72844ac060231b6c01bce9b9919061a81a5a Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 12 Aug 2022 10:08:16 +0900
Subject: [PATCH 0545/3199] save

---
 scheduler/src/lib.rs                    | 2 +-
 transaction-scheduler-bench/src/main.rs | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 8d38e59e008a27..9970759be8e503 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -856,7 +856,7 @@ impl ScheduleStage {
 
                     for vv in vvv {
                         for v in vv {
-                            Self::register_runnable_task((Weight { ix: p }, v), runnable_queue, &mut current_unique_key);
+                            Self::register_runnable_task((Weight { ix: v.2 }, v), runnable_queue, &mut current_unique_key);
                             if executing_queue_count < max_executing_queue_count {
                                 let maybe_ee =
                                     Self::schedule_next_execution(runnable_queue, contended_queue, address_book);
diff --git a/transaction-scheduler-bench/src/main.rs b/transaction-scheduler-bench/src/main.rs
index b5895b0a2d9d27..6b5aa28f233194 100644
--- a/transaction-scheduler-bench/src/main.rs
+++ b/transaction-scheduler-bench/src/main.rs
@@ -78,7 +78,7 @@ struct Args {
 }
 
 /// Some convenient type aliases
-type PreprocessedTransaction = Box<(SanitizedTransaction, Vec<solana_scheduler::LockAttempt>)>;
+type PreprocessedTransaction = Box<(SanitizedTransaction, Vec<solana_scheduler::LockAttempt>, u64)>;
 type TransactionMessage = PreprocessedTransaction;
 type CompletedTransactionMessage = solana_scheduler::Multiplexed; // (usize, Box<solana_scheduler::ExecutionEnvironment>); //(usize, TransactionMessage); // thread index and transaction message
 type TransactionBatchMessage = Box<solana_scheduler::ExecutionEnvironment>; // Vec<TransactionMessage>;

From f894a1bb92416c384b8f950c2fe9782ffd96506f Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 12 Aug 2022 10:08:45 +0900
Subject: [PATCH 0546/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 9970759be8e503..deea0752c8ebbc 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -468,7 +468,7 @@ fn attempt_lock_for_execution<'a>(
     (unlockable_count, guaranteed_count, placeholder_attempts)
 }
 
-type PreprocessedTransaction = (SanitizedTransaction, Vec<LockAttempt>);
+type PreprocessedTransaction = (SanitizedTransaction, Vec<LockAttempt>, u64);
 // multiplexed to reduce the futex syscal per tx down to minimum and to make the schduler to
 // adaptive relative load between sigverify stage and execution substage
 // switched from crossbeam_channel::select! due to observed poor performance

From c282532de0ef011c5ff0157c134f0bdd29858c58 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 12 Aug 2022 10:09:11 +0900
Subject: [PATCH 0547/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index deea0752c8ebbc..f93a0c5e67666c 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -822,7 +822,7 @@ impl ScheduleStage {
 
     #[inline(never)]
     fn register_runnable_task(
-        weighted_tx: (Weight, Box<(SanitizedTransaction, Vec<LockAttempt>)>),
+        weighted_tx: (Weight, Box<(SanitizedTransaction, Vec<LockAttempt>, u64)>),
         runnable_queue: &mut TaskQueue,
         unique_key: &mut u64,
     ) {

From 4268221ad9b4f68abee6c7078845d33362713cb3 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 12 Aug 2022 10:09:24 +0900
Subject: [PATCH 0548/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index f93a0c5e67666c..de1e91f6b80871 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -494,7 +494,7 @@ pub struct ScheduleStage {}
 
 impl ScheduleStage {
     fn push_to_queue(
-        (weight, tx): (Weight, Box<(SanitizedTransaction, Vec<LockAttempt>)>),
+        (weight, tx): (Weight, Box<(SanitizedTransaction, Vec<LockAttempt>, u64)>),
         runnable_queue: &mut TaskQueue,
         unique_key: &mut u64,
     ) {

From d26251a2b0962177f3d88bb8169ab92f971db75b Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 12 Aug 2022 10:10:33 +0900
Subject: [PATCH 0549/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index de1e91f6b80871..c5b5b918282168 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -396,7 +396,7 @@ struct Bundle {
 
 #[derive(Debug)]
 pub struct Task {
-    pub tx: Box<(SanitizedTransaction, Vec<LockAttempt>)>, // actually should be Bundle
+    pub tx: Box<(SanitizedTransaction, Vec<LockAttempt>, u64)>, // actually should be Bundle
     pub contention_count: usize,
 }
 

From d7589ed0b69aa753373b67fe42eb3f7c716eab36 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 12 Aug 2022 10:10:48 +0900
Subject: [PATCH 0550/3199] save

---
 transaction-scheduler-bench/src/main.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/transaction-scheduler-bench/src/main.rs b/transaction-scheduler-bench/src/main.rs
index 6b5aa28f233194..986295714110f8 100644
--- a/transaction-scheduler-bench/src/main.rs
+++ b/transaction-scheduler-bench/src/main.rs
@@ -515,7 +515,7 @@ fn build_packet(
     let locks = writable_lock_iter.chain(readonly_lock_iter).collect::<Vec<_>>();
     let p = solana_scheduler::get_transaction_priority_details(&sanitized_tx);
 
-    Box::new((sanitized_tx, locks))
+    Box::new((sanitized_tx, locks, p))
 }
 
 fn build_accounts(num_accounts: usize) -> Vec<Keypair> {

From 1e37e0e2263828418939ec4fc6c46f957cea2f2a Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 12 Aug 2022 10:16:04 +0900
Subject: [PATCH 0551/3199] save

---
 scheduler/src/lib.rs | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index c5b5b918282168..da3f45c1d87ef8 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -382,7 +382,7 @@ pub struct UniqueWeight {
     // we can't use Transaction::message_hash because it's manipulatable to be favorous to the tx
     // submitter
     //unique_key: Hash, // tie breaker? random noise? also for unique identification of txes?
-    unique_key: u64, // tie breaker? random noise? also for unique identification of txes?
+    //unique_key: u64, // tie breaker? random noise? also for unique identification of txes?
     // fee?
 }
 /*
@@ -509,12 +509,14 @@ impl ScheduleStage {
         //    )
         //    .unwrap();
         //tx.foo();
+        assert!(weight < 0x1_0000_0000);
+        let weight = (weight << 32) | (unique_key >> 32);
 
         runnable_queue.add_to_schedule(
             UniqueWeight {
                 weight,
                 //unique_key: solana_sdk::hash::new_rand(&mut rng),
-                unique_key: *unique_key,
+                //unique_key: *unique_key,
             },
             Task { tx, contention_count: 0 },
         );

From 694e1eef88ae5258bd4b0d972fc31161bf19a723 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 12 Aug 2022 10:16:25 +0900
Subject: [PATCH 0552/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index da3f45c1d87ef8..c4ddd2b349d34e 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -510,7 +510,7 @@ impl ScheduleStage {
         //    .unwrap();
         //tx.foo();
         assert!(weight < 0x1_0000_0000);
-        let weight = (weight << 32) | (unique_key >> 32);
+        let weight = (weight << 32) | (*unique_key >> 32);
 
         runnable_queue.add_to_schedule(
             UniqueWeight {

From 53583686beaa32466a3b5d65803c1f6fb2e2de74 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 12 Aug 2022 10:16:36 +0900
Subject: [PATCH 0553/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index c4ddd2b349d34e..16591cad345b5a 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -510,7 +510,7 @@ impl ScheduleStage {
         //    .unwrap();
         //tx.foo();
         assert!(weight < 0x1_0000_0000);
-        let weight = (weight << 32) | (*unique_key >> 32);
+        let weight = (weight.ix << 32) | (*unique_key >> 32);
 
         runnable_queue.add_to_schedule(
             UniqueWeight {

From 566dddffdf8d6d48c0fd7da1c95bd7f96c28c260 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 12 Aug 2022 10:16:46 +0900
Subject: [PATCH 0554/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 16591cad345b5a..22a02c9afc9dc1 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -509,7 +509,7 @@ impl ScheduleStage {
         //    )
         //    .unwrap();
         //tx.foo();
-        assert!(weight < 0x1_0000_0000);
+        assert!(weight.ix < 0x1_0000_0000);
         let weight = (weight.ix << 32) | (*unique_key >> 32);
 
         runnable_queue.add_to_schedule(

From fdd4d20792eeafc422b369c37875b85be98ee976 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 12 Aug 2022 10:17:02 +0900
Subject: [PATCH 0555/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 22a02c9afc9dc1..57c15435c6a332 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -514,7 +514,7 @@ impl ScheduleStage {
 
         runnable_queue.add_to_schedule(
             UniqueWeight {
-                weight,
+                Weight {ix: weight},
                 //unique_key: solana_sdk::hash::new_rand(&mut rng),
                 //unique_key: *unique_key,
             },

From 5f86e95f9f385b9cb2776395496bb8b39f59d7c4 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 12 Aug 2022 10:17:15 +0900
Subject: [PATCH 0556/3199] save

---
 scheduler/src/lib.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 57c15435c6a332..03b8005335ae1b 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -510,11 +510,11 @@ impl ScheduleStage {
         //    .unwrap();
         //tx.foo();
         assert!(weight.ix < 0x1_0000_0000);
-        let weight = (weight.ix << 32) | (*unique_key >> 32);
+        let weight = Weight { ix: (weight.ix << 32) | (*unique_key >> 32) };
 
         runnable_queue.add_to_schedule(
             UniqueWeight {
-                Weight {ix: weight},
+                weight,
                 //unique_key: solana_sdk::hash::new_rand(&mut rng),
                 //unique_key: *unique_key,
             },

From 53af34b1e6e71c907d57bbdd6f433c7a0c835fbd Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 12 Aug 2022 10:18:22 +0900
Subject: [PATCH 0557/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 03b8005335ae1b..b073636165293f 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -510,7 +510,7 @@ impl ScheduleStage {
         //    .unwrap();
         //tx.foo();
         assert!(weight.ix < 0x1_0000_0000);
-        let weight = Weight { ix: (weight.ix << 32) | (*unique_key >> 32) };
+        let weight = Weight { ix: (weight.ix << 32) | (*unique_key & 0xffff_ffff_0000_0000) };
 
         runnable_queue.add_to_schedule(
             UniqueWeight {

From c62d431c1cde68ab6ddcf3adad91d466e2448ebb Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 12 Aug 2022 10:20:11 +0900
Subject: [PATCH 0558/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index b073636165293f..98db0896cde5e2 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -510,7 +510,7 @@ impl ScheduleStage {
         //    .unwrap();
         //tx.foo();
         assert!(weight.ix < 0x1_0000_0000);
-        let weight = Weight { ix: (weight.ix << 32) | (*unique_key & 0xffff_ffff_0000_0000) };
+        let weight = Weight { ix: (weight.ix << 32) | (*unique_key & 0x0000_0000_ffff_ffff) };
 
         runnable_queue.add_to_schedule(
             UniqueWeight {

From c92084a0216e5f5ace9a5ff46edcc7d0d0b6f269 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 12 Aug 2022 10:22:03 +0900
Subject: [PATCH 0559/3199] save

---
 scheduler/src/lib.rs | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 98db0896cde5e2..79ae16feb1ded5 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -468,13 +468,13 @@ fn attempt_lock_for_execution<'a>(
     (unlockable_count, guaranteed_count, placeholder_attempts)
 }
 
-type PreprocessedTransaction = (SanitizedTransaction, Vec<LockAttempt>, u64);
+type PreloadedTransaction = (SanitizedTransaction, Vec<LockAttempt>, u64);
 // multiplexed to reduce the futex syscal per tx down to minimum and to make the schduler to
 // adaptive relative load between sigverify stage and execution substage
 // switched from crossbeam_channel::select! due to observed poor performance
 pub enum Multiplexed {
-    FromPrevious((Weight, Box<PreprocessedTransaction>)),
-    FromPreviousBatched(Vec<Vec<Box<PreprocessedTransaction>>>),
+    FromPrevious((Weight, Box<PreloadedTransaction>)),
+    FromPreviousBatched(Vec<Vec<Box<PreloadedTransaction>>>),
     FromExecute(Box<ExecutionEnvironment>),
 }
 
@@ -494,7 +494,7 @@ pub struct ScheduleStage {}
 
 impl ScheduleStage {
     fn push_to_queue(
-        (weight, tx): (Weight, Box<(SanitizedTransaction, Vec<LockAttempt>, u64)>),
+        (weight, tx): (Weight, Box<(PreloadedTransaction)>),
         runnable_queue: &mut TaskQueue,
         unique_key: &mut u64,
     ) {
@@ -824,7 +824,7 @@ impl ScheduleStage {
 
     #[inline(never)]
     fn register_runnable_task(
-        weighted_tx: (Weight, Box<(SanitizedTransaction, Vec<LockAttempt>, u64)>),
+        weighted_tx: (Weight, Box<PreloadedTransaction>),
         runnable_queue: &mut TaskQueue,
         unique_key: &mut u64,
     ) {

From b782f4e048eda1e1f226a7e8ac93d65b4697679e Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 12 Aug 2022 10:22:33 +0900
Subject: [PATCH 0560/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 79ae16feb1ded5..e413ac325b965f 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -494,7 +494,7 @@ pub struct ScheduleStage {}
 
 impl ScheduleStage {
     fn push_to_queue(
-        (weight, tx): (Weight, Box<(PreloadedTransaction)>),
+        (weight, tx): (Weight, Box<PreloadedTransaction>),
         runnable_queue: &mut TaskQueue,
         unique_key: &mut u64,
     ) {

From e0c4a60bd0d72c5391e4434add708c3a88992a60 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 12 Aug 2022 14:32:19 +0900
Subject: [PATCH 0561/3199] save

---
 scheduler/src/lib.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index e413ac325b965f..b69131bb8ddd80 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -510,13 +510,13 @@ impl ScheduleStage {
         //    .unwrap();
         //tx.foo();
         assert!(weight.ix < 0x1_0000_0000);
-        let weight = Weight { ix: (weight.ix << 32) | (*unique_key & 0x0000_0000_ffff_ffff) };
+        //let weight = Weight { ix: (weight.ix << 32) | (*unique_key & 0x0000_0000_ffff_ffff) };
 
         runnable_queue.add_to_schedule(
             UniqueWeight {
                 weight,
                 //unique_key: solana_sdk::hash::new_rand(&mut rng),
-                //unique_key: *unique_key,
+                unique_key: *unique_key,
             },
             Task { tx, contention_count: 0 },
         );

From 75c1aed72050fb69153899e913841ab3cc73b089 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 12 Aug 2022 14:32:36 +0900
Subject: [PATCH 0562/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index b69131bb8ddd80..56f8827acad699 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -382,7 +382,7 @@ pub struct UniqueWeight {
     // we can't use Transaction::message_hash because it's manipulatable to be favorous to the tx
     // submitter
     //unique_key: Hash, // tie breaker? random noise? also for unique identification of txes?
-    //unique_key: u64, // tie breaker? random noise? also for unique identification of txes?
+    unique_key: u64, // tie breaker? random noise? also for unique identification of txes?
     // fee?
 }
 /*

From 03dc42de3095aa96e14b107df856dbe845f1e664 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 12 Aug 2022 14:33:39 +0900
Subject: [PATCH 0563/3199] save

---
 scheduler/src/lib.rs | 19 +++++++++----------
 1 file changed, 9 insertions(+), 10 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 56f8827acad699..2727c692368c0b 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -396,7 +396,7 @@ struct Bundle {
 
 #[derive(Debug)]
 pub struct Task {
-    pub tx: Box<(SanitizedTransaction, Vec<LockAttempt>, u64)>, // actually should be Bundle
+    pub tx: Box<(SanitizedTransaction, Vec<LockAttempt>)>, // actually should be Bundle
     pub contention_count: usize,
 }
 
@@ -468,17 +468,17 @@ fn attempt_lock_for_execution<'a>(
     (unlockable_count, guaranteed_count, placeholder_attempts)
 }
 
-type PreloadedTransaction = (SanitizedTransaction, Vec<LockAttempt>, u64);
+type PreprocessedTransaction = (SanitizedTransaction, Vec<LockAttempt>);
 // multiplexed to reduce the futex syscal per tx down to minimum and to make the schduler to
 // adaptive relative load between sigverify stage and execution substage
 // switched from crossbeam_channel::select! due to observed poor performance
 pub enum Multiplexed {
-    FromPrevious((Weight, Box<PreloadedTransaction>)),
-    FromPreviousBatched(Vec<Vec<Box<PreloadedTransaction>>>),
+    FromPrevious((Weight, Box<PreprocessedTransaction>)),
+    FromPreviousBatched(Vec<Vec<Box<PreprocessedTransaction>>>),
     FromExecute(Box<ExecutionEnvironment>),
 }
 
-pub fn get_transaction_priority_details(tx: &SanitizedTransaction) -> u64 {
+fn get_transaction_priority_details(tx: &SanitizedTransaction) -> u64 {
         use solana_program_runtime::compute_budget::ComputeBudget;
         let mut compute_budget = ComputeBudget::default();
         compute_budget
@@ -494,7 +494,7 @@ pub struct ScheduleStage {}
 
 impl ScheduleStage {
     fn push_to_queue(
-        (weight, tx): (Weight, Box<PreloadedTransaction>),
+        (weight, tx): (Weight, Box<(SanitizedTransaction, Vec<LockAttempt>)>),
         runnable_queue: &mut TaskQueue,
         unique_key: &mut u64,
     ) {
@@ -509,8 +509,6 @@ impl ScheduleStage {
         //    )
         //    .unwrap();
         //tx.foo();
-        assert!(weight.ix < 0x1_0000_0000);
-        //let weight = Weight { ix: (weight.ix << 32) | (*unique_key & 0x0000_0000_ffff_ffff) };
 
         runnable_queue.add_to_schedule(
             UniqueWeight {
@@ -824,7 +822,7 @@ impl ScheduleStage {
 
     #[inline(never)]
     fn register_runnable_task(
-        weighted_tx: (Weight, Box<PreloadedTransaction>),
+        weighted_tx: (Weight, Box<(SanitizedTransaction, Vec<LockAttempt>)>),
         runnable_queue: &mut TaskQueue,
         unique_key: &mut u64,
     ) {
@@ -858,7 +856,8 @@ impl ScheduleStage {
 
                     for vv in vvv {
                         for v in vv {
-                            Self::register_runnable_task((Weight { ix: v.2 }, v), runnable_queue, &mut current_unique_key);
+                            let p = get_transaction_priority_details(&v.0);
+                            Self::register_runnable_task((Weight { ix: p }, v), runnable_queue, &mut current_unique_key);
                             if executing_queue_count < max_executing_queue_count {
                                 let maybe_ee =
                                     Self::schedule_next_execution(runnable_queue, contended_queue, address_book);

From 3b04ece0b55e360a6fb375ec8ce3d70190a03c6f Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 12 Aug 2022 14:34:06 +0900
Subject: [PATCH 0564/3199] save

---
 transaction-scheduler-bench/src/main.rs | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/transaction-scheduler-bench/src/main.rs b/transaction-scheduler-bench/src/main.rs
index 986295714110f8..f1ad2602aee6e4 100644
--- a/transaction-scheduler-bench/src/main.rs
+++ b/transaction-scheduler-bench/src/main.rs
@@ -78,7 +78,7 @@ struct Args {
 }
 
 /// Some convenient type aliases
-type PreprocessedTransaction = Box<(SanitizedTransaction, Vec<solana_scheduler::LockAttempt>, u64)>;
+type PreprocessedTransaction = Box<(SanitizedTransaction, Vec<solana_scheduler::LockAttempt>)>;
 type TransactionMessage = PreprocessedTransaction;
 type CompletedTransactionMessage = solana_scheduler::Multiplexed; // (usize, Box<solana_scheduler::ExecutionEnvironment>); //(usize, TransactionMessage); // thread index and transaction message
 type TransactionBatchMessage = Box<solana_scheduler::ExecutionEnvironment>; // Vec<TransactionMessage>;
@@ -513,9 +513,8 @@ fn build_packet(
         .iter()
         .map(|address| solana_scheduler::LockAttempt::new(preloader.load(**address), solana_scheduler::RequestedUsage::Readonly));
     let locks = writable_lock_iter.chain(readonly_lock_iter).collect::<Vec<_>>();
-    let p = solana_scheduler::get_transaction_priority_details(&sanitized_tx);
 
-    Box::new((sanitized_tx, locks, p))
+    Box::new((sanitized_tx, locks))
 }
 
 fn build_accounts(num_accounts: usize) -> Vec<Keypair> {

From 43537355ad28bac50aa68e6072487c1523eaafb0 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 12 Aug 2022 14:34:54 +0900
Subject: [PATCH 0565/3199] save

---
 scheduler/src/lib.rs | 12 ++++--------
 1 file changed, 4 insertions(+), 8 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 2727c692368c0b..4d7f55efa0d573 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -216,8 +216,8 @@ impl AddressBook {
                         }
                         RequestedUsage::Writable => {
                             if from_runnable {
-                                //Self::remember_address_contention(&mut page, unique_weight);
-                                //*remembered = true;
+                                Self::remember_address_contention(&mut page, unique_weight);
+                                *remembered = true;
                                 *status = LockStatus::Failed;
                             } else {
                                 match page.next_usage {
@@ -237,8 +237,8 @@ impl AddressBook {
                     },
                     Usage::Writable => {
                         if from_runnable {
-                            //Self::remember_address_contention(&mut page, unique_weight);
-                            //*remembered = true;
+                            Self::remember_address_contention(&mut page, unique_weight);
+                            *remembered = true;
                             *status = LockStatus::Failed;
                         } else {
                             match page.next_usage {
@@ -729,10 +729,6 @@ impl ScheduleStage {
         from_runnable: bool,
     ) {
         for l in lock_attempts {
-            if from_runnable {
-                AddressBook::remember_address_contention(&mut l.target.page(), unique_weight);
-                l.remembered = true;
-            }
             address_book.reset_lock(l, false);
             //if let Some(uw) = l.target.page().contended_unique_weights.last() {
             //    address_book.uncontended_task_ids.remove(uw);

From dc1ce450a6650db6501f4401a141175c9cd63dca Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 12 Aug 2022 14:37:13 +0900
Subject: [PATCH 0566/3199] save

---
 scheduler/src/lib.rs | 12 ++++++++----
 1 file changed, 8 insertions(+), 4 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 4d7f55efa0d573..2727c692368c0b 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -216,8 +216,8 @@ impl AddressBook {
                         }
                         RequestedUsage::Writable => {
                             if from_runnable {
-                                Self::remember_address_contention(&mut page, unique_weight);
-                                *remembered = true;
+                                //Self::remember_address_contention(&mut page, unique_weight);
+                                //*remembered = true;
                                 *status = LockStatus::Failed;
                             } else {
                                 match page.next_usage {
@@ -237,8 +237,8 @@ impl AddressBook {
                     },
                     Usage::Writable => {
                         if from_runnable {
-                            Self::remember_address_contention(&mut page, unique_weight);
-                            *remembered = true;
+                            //Self::remember_address_contention(&mut page, unique_weight);
+                            //*remembered = true;
                             *status = LockStatus::Failed;
                         } else {
                             match page.next_usage {
@@ -729,6 +729,10 @@ impl ScheduleStage {
         from_runnable: bool,
     ) {
         for l in lock_attempts {
+            if from_runnable {
+                AddressBook::remember_address_contention(&mut l.target.page(), unique_weight);
+                l.remembered = true;
+            }
             address_book.reset_lock(l, false);
             //if let Some(uw) = l.target.page().contended_unique_weights.last() {
             //    address_book.uncontended_task_ids.remove(uw);

From e9b5d7f2fc1c0c7a161772cd99c658a88578831c Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 12 Aug 2022 14:49:53 +0900
Subject: [PATCH 0567/3199] save

---
 scheduler/src/lib.rs | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 2727c692368c0b..132c15fd90d926 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -858,6 +858,7 @@ impl ScheduleStage {
                         for v in vv {
                             let p = get_transaction_priority_details(&v.0);
                             Self::register_runnable_task((Weight { ix: p }, v), runnable_queue, &mut current_unique_key);
+                            /*
                             if executing_queue_count < max_executing_queue_count {
                                 let maybe_ee =
                                     Self::schedule_next_execution(runnable_queue, contended_queue, address_book);
@@ -870,6 +871,7 @@ impl ScheduleStage {
                             } else {
                                     trace!("batched: outgoing queue full");
                             }
+                            */
                         }
                     }
                 }

From f111c848b3cb9f818107b61876b8c50e3fb72fd6 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 12 Aug 2022 14:53:48 +0900
Subject: [PATCH 0568/3199] save

---
 scheduler/src/lib.rs | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 132c15fd90d926..de10ccc7097f74 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -216,8 +216,8 @@ impl AddressBook {
                         }
                         RequestedUsage::Writable => {
                             if from_runnable {
-                                //Self::remember_address_contention(&mut page, unique_weight);
-                                //*remembered = true;
+                                Self::remember_address_contention(&mut page, unique_weight);
+                                *remembered = true;
                                 *status = LockStatus::Failed;
                             } else {
                                 match page.next_usage {
@@ -237,8 +237,8 @@ impl AddressBook {
                     },
                     Usage::Writable => {
                         if from_runnable {
-                            //Self::remember_address_contention(&mut page, unique_weight);
-                            //*remembered = true;
+                            Self::remember_address_contention(&mut page, unique_weight);
+                            *remembered = true;
                             *status = LockStatus::Failed;
                         } else {
                             match page.next_usage {
@@ -730,8 +730,8 @@ impl ScheduleStage {
     ) {
         for l in lock_attempts {
             if from_runnable {
-                AddressBook::remember_address_contention(&mut l.target.page(), unique_weight);
-                l.remembered = true;
+                //AddressBook::remember_address_contention(&mut l.target.page(), unique_weight);
+                //l.remembered = true;
             }
             address_book.reset_lock(l, false);
             //if let Some(uw) = l.target.page().contended_unique_weights.last() {

From e8f30a8c561210c9e43d719e17e6d8ca46abd76d Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 12 Aug 2022 15:07:45 +0900
Subject: [PATCH 0569/3199] save

---
 scheduler/src/lib.rs | 26 ++++++++++++++++++++++----
 1 file changed, 22 insertions(+), 4 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index de10ccc7097f74..a1ac505d433bd5 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -547,6 +547,7 @@ impl ScheduleStage {
         runnable_queue: &'a mut TaskQueue,
         contended_queue: &'a mut TaskQueue,
         address_book: &mut AddressBook,
+        prefer_contend: bool
     ) -> Option<(
         Option<&'a mut TaskQueue>,
         std::collections::btree_map::OccupiedEntry<'a, UniqueWeight, Task>,
@@ -557,7 +558,11 @@ impl ScheduleStage {
         ) {
             (Some(heaviest_runnable_entry), None) => {
                 trace!("select: runnable only");
-                Some((Some(contended_queue), heaviest_runnable_entry))
+                if prefer_contend {
+                    None
+                } else {
+                    Some((Some(contended_queue), heaviest_runnable_entry))
+                }
             }
             (None, Some(weight_from_contended)) => {
                 trace!("select: contended only");
@@ -570,6 +575,16 @@ impl ScheduleStage {
                 let weight_from_runnable = heaviest_runnable_entry.key();
                 let uw = weight_from_contended.key();
 
+                if prefer_contend {
+                    trace!("select: contended > runnnable");
+                    let uw = *uw;
+                    weight_from_contended.remove();
+                    return Some((
+                        None,
+                        contended_queue.entry_to_execute(uw),
+                    ))
+                }
+
                 if weight_from_runnable > uw {
                     trace!("select: runnable > contended");
                     Some((Some(contended_queue), heaviest_runnable_entry))
@@ -599,6 +614,7 @@ impl ScheduleStage {
         runnable_queue: &mut TaskQueue,
         contended_queue: &mut TaskQueue,
         address_book: &mut AddressBook,
+        prefer_contend: bool,
     ) -> Option<(UniqueWeight, Task, Vec<LockAttempt>)> {
         if let Some(a) = address_book.runnable_guaranteed_task_ids.pop_last() {
             trace!("expediate pop from guaranteed queue [rest: {}]", address_book.runnable_guaranteed_task_ids.len());
@@ -610,7 +626,7 @@ impl ScheduleStage {
 
         trace!("pop begin");
         loop {
-        if let Some((reborrowed_contended_queue, mut queue_entry)) = Self::select_next_task(runnable_queue, contended_queue, address_book) {
+        if let Some((reborrowed_contended_queue, mut queue_entry)) = Self::select_next_task(runnable_queue, contended_queue, address_book, prefer_contend) {
             trace!("pop loop iteration");
             let from_runnable = reborrowed_contended_queue.is_some();
             let unique_weight = *queue_entry.key();
@@ -813,9 +829,10 @@ impl ScheduleStage {
         runnable_queue: &mut TaskQueue,
         contended_queue: &mut TaskQueue,
         address_book: &mut AddressBook,
+        prefer_contend: bool,
     ) -> Option<Box<ExecutionEnvironment>> {
         let maybe_ee =
-            Self::pop_from_queue_then_lock(runnable_queue, contended_queue, address_book)
+            Self::pop_from_queue_then_lock(runnable_queue, contended_queue, address_book, prefer_contend)
                 .map(|(uw, t, ll)| Self::prepare_scheduled_execution(address_book, uw, t, ll));
         maybe_ee
     }
@@ -895,8 +912,9 @@ impl ScheduleStage {
                     break;
                 }
 
+                let prefer_contend = executing_queue_count == max_executing_queue_count;
                 let maybe_ee =
-                    Self::schedule_next_execution(runnable_queue, contended_queue, address_book);
+                    Self::schedule_next_execution(runnable_queue, contended_queue, address_book, prefer_contend);
 
                 if let Some(ee) = maybe_ee {
                     trace!("send to execute");

From 70c350dba37798a4d5d4635a662f667c57dbccad Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 12 Aug 2022 15:28:29 +0900
Subject: [PATCH 0570/3199] save

---
 transaction-scheduler-bench/src/main.rs | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/transaction-scheduler-bench/src/main.rs b/transaction-scheduler-bench/src/main.rs
index f1ad2602aee6e4..b72f595ec08a9b 100644
--- a/transaction-scheduler-bench/src/main.rs
+++ b/transaction-scheduler-bench/src/main.rs
@@ -441,7 +441,13 @@ fn send_packets(
             packet_batches.iter().map(|pb| pb.len()).sum(),
             Ordering::Relaxed,
         );
-        let _ = packet_batch_sender.send(solana_scheduler::Multiplexed::FromPreviousBatched(packet_batches));
+
+        for vv in vvv {
+            for v in vv {
+                packet_batch_sender.send(solana_scheduler::Multiplexed::FromPrevious(v));
+            }
+        }
+        
 
         std::thread::sleep(loop_duration.saturating_sub(packet_build_time.as_duration()));
     }

From cd6dafa58dfb68e1c7cc5d219699d85a66b26afc Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 12 Aug 2022 15:28:52 +0900
Subject: [PATCH 0571/3199] save

---
 transaction-scheduler-bench/src/main.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/transaction-scheduler-bench/src/main.rs b/transaction-scheduler-bench/src/main.rs
index b72f595ec08a9b..6a70b2d0b96b66 100644
--- a/transaction-scheduler-bench/src/main.rs
+++ b/transaction-scheduler-bench/src/main.rs
@@ -442,7 +442,7 @@ fn send_packets(
             Ordering::Relaxed,
         );
 
-        for vv in vvv {
+        for vv in packet_batches {
             for v in vv {
                 packet_batch_sender.send(solana_scheduler::Multiplexed::FromPrevious(v));
             }

From 435e14f7c2bbef0f7f90d4131e198b4d06782389 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 12 Aug 2022 15:29:39 +0900
Subject: [PATCH 0572/3199] save

---
 scheduler/src/lib.rs                    | 28 +++++--------------------
 transaction-scheduler-bench/src/main.rs |  3 ++-
 2 files changed, 7 insertions(+), 24 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index a1ac505d433bd5..f682ddf6635db5 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -478,7 +478,7 @@ pub enum Multiplexed {
     FromExecute(Box<ExecutionEnvironment>),
 }
 
-fn get_transaction_priority_details(tx: &SanitizedTransaction) -> u64 {
+pub fn get_transaction_priority_details(tx: &SanitizedTransaction) -> u64 {
         use solana_program_runtime::compute_budget::ComputeBudget;
         let mut compute_budget = ComputeBudget::default();
         compute_budget
@@ -547,7 +547,6 @@ impl ScheduleStage {
         runnable_queue: &'a mut TaskQueue,
         contended_queue: &'a mut TaskQueue,
         address_book: &mut AddressBook,
-        prefer_contend: bool
     ) -> Option<(
         Option<&'a mut TaskQueue>,
         std::collections::btree_map::OccupiedEntry<'a, UniqueWeight, Task>,
@@ -558,11 +557,7 @@ impl ScheduleStage {
         ) {
             (Some(heaviest_runnable_entry), None) => {
                 trace!("select: runnable only");
-                if prefer_contend {
-                    None
-                } else {
-                    Some((Some(contended_queue), heaviest_runnable_entry))
-                }
+                Some((Some(contended_queue), heaviest_runnable_entry))
             }
             (None, Some(weight_from_contended)) => {
                 trace!("select: contended only");
@@ -575,16 +570,6 @@ impl ScheduleStage {
                 let weight_from_runnable = heaviest_runnable_entry.key();
                 let uw = weight_from_contended.key();
 
-                if prefer_contend {
-                    trace!("select: contended > runnnable");
-                    let uw = *uw;
-                    weight_from_contended.remove();
-                    return Some((
-                        None,
-                        contended_queue.entry_to_execute(uw),
-                    ))
-                }
-
                 if weight_from_runnable > uw {
                     trace!("select: runnable > contended");
                     Some((Some(contended_queue), heaviest_runnable_entry))
@@ -614,7 +599,6 @@ impl ScheduleStage {
         runnable_queue: &mut TaskQueue,
         contended_queue: &mut TaskQueue,
         address_book: &mut AddressBook,
-        prefer_contend: bool,
     ) -> Option<(UniqueWeight, Task, Vec<LockAttempt>)> {
         if let Some(a) = address_book.runnable_guaranteed_task_ids.pop_last() {
             trace!("expediate pop from guaranteed queue [rest: {}]", address_book.runnable_guaranteed_task_ids.len());
@@ -626,7 +610,7 @@ impl ScheduleStage {
 
         trace!("pop begin");
         loop {
-        if let Some((reborrowed_contended_queue, mut queue_entry)) = Self::select_next_task(runnable_queue, contended_queue, address_book, prefer_contend) {
+        if let Some((reborrowed_contended_queue, mut queue_entry)) = Self::select_next_task(runnable_queue, contended_queue, address_book) {
             trace!("pop loop iteration");
             let from_runnable = reborrowed_contended_queue.is_some();
             let unique_weight = *queue_entry.key();
@@ -829,10 +813,9 @@ impl ScheduleStage {
         runnable_queue: &mut TaskQueue,
         contended_queue: &mut TaskQueue,
         address_book: &mut AddressBook,
-        prefer_contend: bool,
     ) -> Option<Box<ExecutionEnvironment>> {
         let maybe_ee =
-            Self::pop_from_queue_then_lock(runnable_queue, contended_queue, address_book, prefer_contend)
+            Self::pop_from_queue_then_lock(runnable_queue, contended_queue, address_book)
                 .map(|(uw, t, ll)| Self::prepare_scheduled_execution(address_book, uw, t, ll));
         maybe_ee
     }
@@ -912,9 +895,8 @@ impl ScheduleStage {
                     break;
                 }
 
-                let prefer_contend = executing_queue_count == max_executing_queue_count;
                 let maybe_ee =
-                    Self::schedule_next_execution(runnable_queue, contended_queue, address_book, prefer_contend);
+                    Self::schedule_next_execution(runnable_queue, contended_queue, address_book);
 
                 if let Some(ee) = maybe_ee {
                     trace!("send to execute");
diff --git a/transaction-scheduler-bench/src/main.rs b/transaction-scheduler-bench/src/main.rs
index 6a70b2d0b96b66..5bf66d93a56a29 100644
--- a/transaction-scheduler-bench/src/main.rs
+++ b/transaction-scheduler-bench/src/main.rs
@@ -444,7 +444,8 @@ fn send_packets(
 
         for vv in packet_batches {
             for v in vv {
-                packet_batch_sender.send(solana_scheduler::Multiplexed::FromPrevious(v));
+                let p = get_transaction_priority_details(&v.0);
+                packet_batch_sender.send(solana_scheduler::Multiplexed::FromPrevious((Weight { ix: p }, v)));
             }
         }
         

From a50b39e3bf69cb91ca527c4c3d9140c0e04a3bb9 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 12 Aug 2022 15:30:02 +0900
Subject: [PATCH 0573/3199] save

---
 transaction-scheduler-bench/src/main.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/transaction-scheduler-bench/src/main.rs b/transaction-scheduler-bench/src/main.rs
index 5bf66d93a56a29..7351e4213a3c86 100644
--- a/transaction-scheduler-bench/src/main.rs
+++ b/transaction-scheduler-bench/src/main.rs
@@ -444,8 +444,8 @@ fn send_packets(
 
         for vv in packet_batches {
             for v in vv {
-                let p = get_transaction_priority_details(&v.0);
-                packet_batch_sender.send(solana_scheduler::Multiplexed::FromPrevious((Weight { ix: p }, v)));
+                let p = solana_scheduler::get_transaction_priority_details(&v.0);
+                packet_batch_sender.send(solana_scheduler::Multiplexed::FromPrevious((solana_scheduler::Weight { ix: p }, v)));
             }
         }
         

From 81067a83f6bf89865f2200fd6692ab0ccec66085 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 12 Aug 2022 15:30:15 +0900
Subject: [PATCH 0574/3199] save

---
 transaction-scheduler-bench/src/main.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/transaction-scheduler-bench/src/main.rs b/transaction-scheduler-bench/src/main.rs
index 7351e4213a3c86..b0e5fefec22405 100644
--- a/transaction-scheduler-bench/src/main.rs
+++ b/transaction-scheduler-bench/src/main.rs
@@ -445,7 +445,7 @@ fn send_packets(
         for vv in packet_batches {
             for v in vv {
                 let p = solana_scheduler::get_transaction_priority_details(&v.0);
-                packet_batch_sender.send(solana_scheduler::Multiplexed::FromPrevious((solana_scheduler::Weight { ix: p }, v)));
+                packet_batch_sender.send(solana_scheduler::Multiplexed::FromPrevious((solana_scheduler::Weight { ix: p }, v))).unwrap();
             }
         }
         

From 1808b6cf619f15b9112e0d12f97448260eeb51f8 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 12 Aug 2022 15:34:26 +0900
Subject: [PATCH 0575/3199] save

---
 scheduler/src/lib.rs | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index f682ddf6635db5..b614565628a071 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -216,8 +216,8 @@ impl AddressBook {
                         }
                         RequestedUsage::Writable => {
                             if from_runnable {
-                                Self::remember_address_contention(&mut page, unique_weight);
-                                *remembered = true;
+                                //Self::remember_address_contention(&mut page, unique_weight);
+                                //*remembered = true;
                                 *status = LockStatus::Failed;
                             } else {
                                 match page.next_usage {
@@ -237,8 +237,8 @@ impl AddressBook {
                     },
                     Usage::Writable => {
                         if from_runnable {
-                            Self::remember_address_contention(&mut page, unique_weight);
-                            *remembered = true;
+                            //Self::remember_address_contention(&mut page, unique_weight);
+                            //*remembered = true;
                             *status = LockStatus::Failed;
                         } else {
                             match page.next_usage {
@@ -730,8 +730,8 @@ impl ScheduleStage {
     ) {
         for l in lock_attempts {
             if from_runnable {
-                //AddressBook::remember_address_contention(&mut l.target.page(), unique_weight);
-                //l.remembered = true;
+                AddressBook::remember_address_contention(&mut l.target.page(), unique_weight);
+                l.remembered = true;
             }
             address_book.reset_lock(l, false);
             //if let Some(uw) = l.target.page().contended_unique_weights.last() {

From 136da4284ba218a88372846a22ae4bff03f62fc2 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 12 Aug 2022 21:56:51 +0900
Subject: [PATCH 0576/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index b614565628a071..4ebf5ea5afcd5e 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -842,7 +842,7 @@ impl ScheduleStage {
         let mut current_unique_key = u64::max_value();
 
         loop {
-            trace!("schedule_once (runnnable: {}, contended: {}, guaranteed: {}, exec: {})!", runnable_queue.task_count(), contended_queue.task_count(), address_book.guaranteed_lock_counts.len(), executing_queue_count);
+            trace!("schedule_once (runnnable: {}, contended: {}, guaranteed: {}, exec: {}/{})!", runnable_queue.task_count(), contended_queue.task_count(), address_book.guaranteed_lock_counts.len(), executing_queue_count, max_executing_queue_count);
 
             let i = from.recv().unwrap();
             match i {

From 485b5610a73a9def8278ef4a4350feaa8d394271 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 12 Aug 2022 22:01:16 +0900
Subject: [PATCH 0577/3199] save

---
 scheduler/src/lib.rs | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 4ebf5ea5afcd5e..56605bf5916b40 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -164,6 +164,7 @@ impl Page {
 
     fn switch_to_next_usage(&mut self) {
         self.current_usage = self.next_usage;
+        self.next_usage = Usage::Unused;
     }
 }
 
@@ -275,10 +276,10 @@ impl AddressBook {
                 self.unlock(attempt)
             },
             LockStatus::Guaranteed => {
-                self.cancel(attempt);
                 if after_execution {
                     self.unlock(attempt)
                 } else {
+                    self.cancel(attempt);
                     false
                 }
             }

From c909ec5a9cf4405d324a3e62c66c697654330dca Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 12 Aug 2022 22:30:13 +0900
Subject: [PATCH 0578/3199] save

---
 scheduler/src/lib.rs | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 56605bf5916b40..975aae37c96768 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -182,7 +182,7 @@ pub struct AddressBook {
     book: AddressMap,
     uncontended_task_ids: WeightedTaskIds,
     runnable_guaranteed_task_ids: WeightedTaskIds,
-    guaranteed_lock_counts: std::collections::HashMap<UniqueWeight, usize>, 
+    gurantee_timers: std::collections::HashMap<UniqueWeight, usize>, 
 }
 
 impl AddressBook {
@@ -665,7 +665,7 @@ impl ScheduleStage {
                 );
                 std::mem::swap(&mut next_task.tx.1, &mut populated_lock_attempts);
 
-                continue;
+                return None;
             }
 
             trace!("successful lock: (from_runnable: {}) after {} contentions", from_runnable, next_task.contention_count);
@@ -718,8 +718,8 @@ impl ScheduleStage {
                 }
             }
         }
-        address_book.guaranteed_lock_counts.insert(*unique_weight, guaranteed_count);
-        trace!("guaranteed_lock_counts: {}", address_book.guaranteed_lock_counts.len());
+        address_book.gurantee_timers.insert(*unique_weight, guaranteed_count);
+        trace!("gurantee_timers: {}", address_book.gurantee_timers.len());
     }
 
     #[inline(never)]
@@ -757,7 +757,7 @@ impl ScheduleStage {
             if page.current_usage == Usage::Unused && page.next_usage != Usage::Unused {
                 page.switch_to_next_usage();
                 for task_id in std::mem::take(&mut page.guaranteed_task_ids).keys() {
-                    match address_book.guaranteed_lock_counts.entry(*task_id) {
+                    match address_book.gurantee_timers.entry(*task_id) {
                         std::collections::hash_map::Entry::Occupied(mut entry) => {
                             let count = entry.get_mut();
                             trace!("guaranteed lock decrease: {} => {}", *count, *count -1);
@@ -843,7 +843,7 @@ impl ScheduleStage {
         let mut current_unique_key = u64::max_value();
 
         loop {
-            trace!("schedule_once (runnnable: {}, contended: {}, guaranteed: {}, exec: {}/{})!", runnable_queue.task_count(), contended_queue.task_count(), address_book.guaranteed_lock_counts.len(), executing_queue_count, max_executing_queue_count);
+            trace!("schedule_once (runnnable: {}, contended: {}, guaranteed: {}, exec: {}/{})!", runnable_queue.task_count(), contended_queue.task_count(), address_book.gurantee_timers.len(), executing_queue_count, max_executing_queue_count);
 
             let i = from.recv().unwrap();
             match i {

From 0b54fc838d3a60ac1ad77d7e1198afbe1b1c5cf8 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 12 Aug 2022 22:33:10 +0900
Subject: [PATCH 0579/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 975aae37c96768..47c973e78dc896 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -891,7 +891,7 @@ impl ScheduleStage {
             loop {
                 /*if !address_book.uncontended_task_ids.is_empty() {
                     trace!("prefer emptying n_u_a");
-                } else */ if executing_queue_count >= max_executing_queue_count {
+                } else */ if (executing_queue_count + address_book.gurantee_timers.len()) >= max_executing_queue_count {
                     trace!("outgoing queue full");
                     break;
                 }

From 3935a5c2124e96e149da3fe64c3fdc437540a1b2 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 12 Aug 2022 22:37:12 +0900
Subject: [PATCH 0580/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 47c973e78dc896..71beb41c07f2bc 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -843,7 +843,7 @@ impl ScheduleStage {
         let mut current_unique_key = u64::max_value();
 
         loop {
-            trace!("schedule_once (runnnable: {}, contended: {}, guaranteed: {}, exec: {}/{})!", runnable_queue.task_count(), contended_queue.task_count(), address_book.gurantee_timers.len(), executing_queue_count, max_executing_queue_count);
+            trace!("schedule_once (runnnable: {}, contended: {}, (guaranteed+exec)/max: ({}+{})/{})!", runnable_queue.task_count(), contended_queue.task_count(), address_book.gurantee_timers.len(), executing_queue_count, max_executing_queue_count);
 
             let i = from.recv().unwrap();
             match i {

From 9ae6a0f1b4fd2109f7c9e6ebdb4d2663829f296e Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 12 Aug 2022 22:37:43 +0900
Subject: [PATCH 0581/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 71beb41c07f2bc..01d8c48a2f0184 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -843,7 +843,7 @@ impl ScheduleStage {
         let mut current_unique_key = u64::max_value();
 
         loop {
-            trace!("schedule_once (runnnable: {}, contended: {}, (guaranteed+exec)/max: ({}+{})/{})!", runnable_queue.task_count(), contended_queue.task_count(), address_book.gurantee_timers.len(), executing_queue_count, max_executing_queue_count);
+            trace!("schedule_once (runnnable: {}, contended: {}, (immediate+guaranteed)/max: ({}+{})/{})!", runnable_queue.task_count(), contended_queue.task_count(), executing_queue_count, address_book.gurantee_timers.len(), max_executing_queue_count);
 
             let i = from.recv().unwrap();
             match i {

From 0947c1fe439e8775509a1c742f1c7dac6583a51e Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 12 Aug 2022 22:54:34 +0900
Subject: [PATCH 0582/3199] save

---
 scheduler/src/lib.rs | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 01d8c48a2f0184..41d8c52039feac 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -665,7 +665,8 @@ impl ScheduleStage {
                 );
                 std::mem::swap(&mut next_task.tx.1, &mut populated_lock_attempts);
 
-                return None;
+                //return None;
+                continue;
             }
 
             trace!("successful lock: (from_runnable: {}) after {} contentions", from_runnable, next_task.contention_count);

From b814eab283e9d3c51fe1884d8162e8603e1360b2 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 12 Aug 2022 22:55:41 +0900
Subject: [PATCH 0583/3199] save

---
 scheduler/src/lib.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 41d8c52039feac..16b6d741798101 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -665,8 +665,8 @@ impl ScheduleStage {
                 );
                 std::mem::swap(&mut next_task.tx.1, &mut populated_lock_attempts);
 
-                //return None;
-                continue;
+                return None;
+                //continue;
             }
 
             trace!("successful lock: (from_runnable: {}) after {} contentions", from_runnable, next_task.contention_count);

From eba56fcce2beeedafbcd61822818b3927f02fd83 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 12 Aug 2022 22:57:26 +0900
Subject: [PATCH 0584/3199] save

---
 scheduler/src/lib.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 16b6d741798101..ceb7db657cc2f3 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -571,10 +571,10 @@ impl ScheduleStage {
                 let weight_from_runnable = heaviest_runnable_entry.key();
                 let uw = weight_from_contended.key();
 
-                if weight_from_runnable > uw {
+                if false && weight_from_runnable > uw {
                     trace!("select: runnable > contended");
                     Some((Some(contended_queue), heaviest_runnable_entry))
-                } else if uw > weight_from_runnable {
+                } else if true || uw > weight_from_runnable {
                     trace!("select: contended > runnnable");
                     let uw = *uw;
                     weight_from_contended.remove();

From 508c5ce2a537ea17b7819ce69332053febf1990e Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 12 Aug 2022 22:57:54 +0900
Subject: [PATCH 0585/3199] save

---
 scheduler/src/lib.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index ceb7db657cc2f3..90f5b1c7b3a660 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -571,10 +571,10 @@ impl ScheduleStage {
                 let weight_from_runnable = heaviest_runnable_entry.key();
                 let uw = weight_from_contended.key();
 
-                if false && weight_from_runnable > uw {
+                if true || weight_from_runnable > uw {
                     trace!("select: runnable > contended");
                     Some((Some(contended_queue), heaviest_runnable_entry))
-                } else if true || uw > weight_from_runnable {
+                } else if false && uw > weight_from_runnable {
                     trace!("select: contended > runnnable");
                     let uw = *uw;
                     weight_from_contended.remove();

From f7a75da53c32ef1bb133b6e3afb055926f88f76b Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 13 Aug 2022 22:08:03 +0900
Subject: [PATCH 0586/3199] save

---
 scheduler/src/lib.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 90f5b1c7b3a660..093cb721f04cce 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -376,6 +376,7 @@ pub struct Weight {
                    // gas fee
 }
 
+/*
 #[derive(Copy, Clone, Debug, Hash, PartialEq, Eq, PartialOrd, Ord)]
 pub struct UniqueWeight {
     // naming: Sequence Ordering?
@@ -386,10 +387,9 @@ pub struct UniqueWeight {
     unique_key: u64, // tie breaker? random noise? also for unique identification of txes?
     // fee?
 }
-/*
+*/
 pub type Weight = usize;
 pub type UniqueWeight = usize;
-*/
 
 struct Bundle {
     // what about bundle1{tx1a, tx2} and bundle2{tx1b, tx2}?

From 57480f0ff1672cf284bc462acec3dac3636c1f33 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 13 Aug 2022 22:08:28 +0900
Subject: [PATCH 0587/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 093cb721f04cce..25678bd8933796 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -369,6 +369,7 @@ impl Preloader {
     }
 }
 
+/*
 #[derive(Copy, Clone, Debug, Hash, PartialEq, Eq, PartialOrd, Ord)]
 pub struct Weight {
     // naming: Sequence Ordering?
@@ -376,7 +377,6 @@ pub struct Weight {
                    // gas fee
 }
 
-/*
 #[derive(Copy, Clone, Debug, Hash, PartialEq, Eq, PartialOrd, Ord)]
 pub struct UniqueWeight {
     // naming: Sequence Ordering?

From ecb17db1b6b62a860d3ead72e2b65deb96477d51 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 13 Aug 2022 22:09:11 +0900
Subject: [PATCH 0588/3199] save

---
 scheduler/src/lib.rs | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 25678bd8933796..4297d7a981e61b 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -510,13 +510,17 @@ impl ScheduleStage {
         //    )
         //    .unwrap();
         //tx.foo();
+        let unique_weight = 0;
 
         runnable_queue.add_to_schedule(
+            /*
             UniqueWeight {
                 weight,
                 //unique_key: solana_sdk::hash::new_rand(&mut rng),
                 unique_key: *unique_key,
             },
+            */
+            unique_weight,
             Task { tx, contention_count: 0 },
         );
         *unique_key -= 1;

From c7af4fc8690134d94cffb6feb16022f5f7c05add Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 13 Aug 2022 22:09:31 +0900
Subject: [PATCH 0589/3199] save

---
 scheduler/src/lib.rs | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 4297d7a981e61b..db3082c4cda910 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -862,8 +862,9 @@ impl ScheduleStage {
 
                     for vv in vvv {
                         for v in vv {
-                            let p = get_transaction_priority_details(&v.0);
-                            Self::register_runnable_task((Weight { ix: p }, v), runnable_queue, &mut current_unique_key);
+                            panic!();
+                            //let p = get_transaction_priority_details(&v.0);
+                            //Self::register_runnable_task((Weight { ix: p }, v), runnable_queue, &mut current_unique_key);
                             /*
                             if executing_queue_count < max_executing_queue_count {
                                 let maybe_ee =

From d5020ab1146f24cef62ffedde5617388572ce546 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 13 Aug 2022 22:10:17 +0900
Subject: [PATCH 0590/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index db3082c4cda910..03ad5e8e0c8928 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -510,7 +510,7 @@ impl ScheduleStage {
         //    )
         //    .unwrap();
         //tx.foo();
-        let unique_weight = 0;
+        let unique_weight = (weight.ix << 32) | (*unique_key & 0x0000_0000_ffff_ffff);
 
         runnable_queue.add_to_schedule(
             /*

From 05511d4b11aac3bbf0b3cb87a6456c735082f774 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 13 Aug 2022 22:10:26 +0900
Subject: [PATCH 0591/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 03ad5e8e0c8928..875b3a7ac78b23 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -510,7 +510,7 @@ impl ScheduleStage {
         //    )
         //    .unwrap();
         //tx.foo();
-        let unique_weight = (weight.ix << 32) | (*unique_key & 0x0000_0000_ffff_ffff);
+        let unique_weight = (weight << 32) | (*unique_key & 0x0000_0000_ffff_ffff);
 
         runnable_queue.add_to_schedule(
             /*

From 17344659c0a2b7ea466f6420e6f65b8ddab40b5c Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 13 Aug 2022 22:10:52 +0900
Subject: [PATCH 0592/3199] save

---
 scheduler/src/lib.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 875b3a7ac78b23..eafdd47d0eb7df 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -388,8 +388,8 @@ pub struct UniqueWeight {
     // fee?
 }
 */
-pub type Weight = usize;
-pub type UniqueWeight = usize;
+pub type Weight = u64;
+pub type UniqueWeight = u64;
 
 struct Bundle {
     // what about bundle1{tx1a, tx2} and bundle2{tx1b, tx2}?

From b16ba80f4b98db16894831d847e47a9839c7259d Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 13 Aug 2022 22:11:19 +0900
Subject: [PATCH 0593/3199] save

---
 transaction-scheduler-bench/src/main.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/transaction-scheduler-bench/src/main.rs b/transaction-scheduler-bench/src/main.rs
index b0e5fefec22405..0a63a0df3da824 100644
--- a/transaction-scheduler-bench/src/main.rs
+++ b/transaction-scheduler-bench/src/main.rs
@@ -445,7 +445,7 @@ fn send_packets(
         for vv in packet_batches {
             for v in vv {
                 let p = solana_scheduler::get_transaction_priority_details(&v.0);
-                packet_batch_sender.send(solana_scheduler::Multiplexed::FromPrevious((solana_scheduler::Weight { ix: p }, v))).unwrap();
+                packet_batch_sender.send(solana_scheduler::Multiplexed::FromPrevious((p, v))).unwrap();
             }
         }
         

From 774d74761e1f0fa2f637e146a3c58e49f42cc782 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 13 Aug 2022 22:27:41 +0900
Subject: [PATCH 0594/3199] save

---
 scheduler/src/lib.rs | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index eafdd47d0eb7df..3ab489f1bba727 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -141,11 +141,15 @@ pub enum RequestedUsage {
     Writable,
 }
 
+pub struct TaskIds {
+    task_ids: std::collections::BTreeSet<UniqueWeight>,
+}
+
 #[derive(Debug)]
 struct Page {
     current_usage: Usage,
     next_usage: Usage,
-    contended_unique_weights: std::collections::BTreeSet<UniqueWeight>,
+    contended_unique_weights: TaskIds,
     guaranteed_task_ids: WeightedTaskIds,
     //loaded account from Accounts db
     //comulative_cu for qos; i.e. track serialized cumulative keyed by addresses and bail out block

From 5abb42807647a4ca8ef8b601443389976cf0eaba Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 13 Aug 2022 22:27:59 +0900
Subject: [PATCH 0595/3199] save

---
 scheduler/src/lib.rs | 1 +
 1 file changed, 1 insertion(+)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 3ab489f1bba727..73245d76107486 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -141,6 +141,7 @@ pub enum RequestedUsage {
     Writable,
 }
 
+#[derive(Debug)]
 pub struct TaskIds {
     task_ids: std::collections::BTreeSet<UniqueWeight>,
 }

From da5b425a500fa37076b7013af6e6ac9cdc37042a Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 13 Aug 2022 22:28:24 +0900
Subject: [PATCH 0596/3199] save

---
 scheduler/src/lib.rs | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 73245d76107486..c01e896f3b179b 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -141,11 +141,14 @@ pub enum RequestedUsage {
     Writable,
 }
 
-#[derive(Debug)]
+#[derive(Debug, Default)]
 pub struct TaskIds {
     task_ids: std::collections::BTreeSet<UniqueWeight>,
 }
 
+impl TaskIds {
+}
+
 #[derive(Debug)]
 struct Page {
     current_usage: Usage,

From 517d2b8374414f848c1d858343687b9be274fb90 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 13 Aug 2022 22:29:01 +0900
Subject: [PATCH 0597/3199] save

---
 scheduler/src/lib.rs | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index c01e896f3b179b..7f701f59ef0137 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -147,6 +147,9 @@ pub struct TaskIds {
 }
 
 impl TaskIds {
+    fn insert(&self, u: usize) -> usize {
+        self.task_ids.insert(u)
+    }
 }
 
 #[derive(Debug)]

From 1a753e4a984ea6c9d17388e2c76a4b4517431c83 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 13 Aug 2022 22:29:22 +0900
Subject: [PATCH 0598/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 7f701f59ef0137..d28b3ce5037ccd 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -147,7 +147,7 @@ pub struct TaskIds {
 }
 
 impl TaskIds {
-    fn insert(&self, u: usize) -> usize {
+    fn insert(&self, u: UniqueWeight) -> usize {
         self.task_ids.insert(u)
     }
 }

From 419dfeddc52d2d94a94621213a97eeaad5a5a08b Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 13 Aug 2022 22:29:35 +0900
Subject: [PATCH 0599/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index d28b3ce5037ccd..da99133f9409e5 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -147,7 +147,7 @@ pub struct TaskIds {
 }
 
 impl TaskIds {
-    fn insert(&self, u: UniqueWeight) -> usize {
+    fn insert(&self, u: UniqueWeight) -> bool {
         self.task_ids.insert(u)
     }
 }

From e7dfb328e0abe241e6ff781652cf5b2fa3c292a3 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 13 Aug 2022 22:30:13 +0900
Subject: [PATCH 0600/3199] save

---
 scheduler/src/lib.rs | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index da99133f9409e5..5675acef25ec3c 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -150,6 +150,10 @@ impl TaskIds {
     fn insert(&self, u: UniqueWeight) -> bool {
         self.task_ids.insert(u)
     }
+
+    fn remov(&self, u: &UniqueWeight) -> usize {
+        self.task_ids.remove(u)
+    }
 }
 
 #[derive(Debug)]

From c4e09c476c0ba5a3050d0666b5dcbfedb2a9f65b Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 13 Aug 2022 22:30:41 +0900
Subject: [PATCH 0601/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 5675acef25ec3c..5a799384a0f259 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -151,7 +151,7 @@ impl TaskIds {
         self.task_ids.insert(u)
     }
 
-    fn remov(&self, u: &UniqueWeight) -> usize {
+    fn remove(&self, u: &UniqueWeight) -> UniqueWeight {
         self.task_ids.remove(u)
     }
 }

From bdbbfb7f51ab3c1ff1176a6f02a2f68499b887ca Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 13 Aug 2022 22:30:57 +0900
Subject: [PATCH 0602/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 5a799384a0f259..f680c17ca9832b 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -151,7 +151,7 @@ impl TaskIds {
         self.task_ids.insert(u)
     }
 
-    fn remove(&self, u: &UniqueWeight) -> UniqueWeight {
+    fn remove(&self, u: &UniqueWeight) -> bool {
         self.task_ids.remove(u)
     }
 }

From 84bbf62278ad2977ee324953be566134acdcb122 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 13 Aug 2022 22:31:22 +0900
Subject: [PATCH 0603/3199] save

---
 scheduler/src/lib.rs | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index f680c17ca9832b..592f754b85485d 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -154,6 +154,10 @@ impl TaskIds {
     fn remove(&self, u: &UniqueWeight) -> bool {
         self.task_ids.remove(u)
     }
+
+    fn is_empty(&self) -> bool {
+        self.task_ids.is_empty()
+    }
 }
 
 #[derive(Debug)]

From 0c6efde749f2b2bdbc4534613a94c32f62cd5042 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 13 Aug 2022 22:31:33 +0900
Subject: [PATCH 0604/3199] save

---
 scheduler/src/lib.rs | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 592f754b85485d..d4554b634d89ef 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -158,6 +158,10 @@ impl TaskIds {
     fn is_empty(&self) -> bool {
         self.task_ids.is_empty()
     }
+
+    fn last(&self) -> bool {
+        self.task_ids.last()
+    }
 }
 
 #[derive(Debug)]

From b46f23057b5eb17fd88845f194f82108489f3baa Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 13 Aug 2022 22:31:45 +0900
Subject: [PATCH 0605/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index d4554b634d89ef..6cf39ccb108156 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -159,7 +159,7 @@ impl TaskIds {
         self.task_ids.is_empty()
     }
 
-    fn last(&self) -> bool {
+    fn last(&self) -> Option<bool> {
         self.task_ids.last()
     }
 }

From cdcaa91faedc4944ebd16a783674226136354214 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 13 Aug 2022 22:31:57 +0900
Subject: [PATCH 0606/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 6cf39ccb108156..09ad7206759416 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -159,7 +159,7 @@ impl TaskIds {
         self.task_ids.is_empty()
     }
 
-    fn last(&self) -> Option<bool> {
+    fn last(&self) -> Option<&UniqueWeight> {
         self.task_ids.last()
     }
 }

From 0aaffb4d79c43989407043d12f83ad3296d13066 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 13 Aug 2022 22:32:22 +0900
Subject: [PATCH 0607/3199] save

---
 scheduler/src/lib.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 09ad7206759416..9e9d5f7eeaded5 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -147,11 +147,11 @@ pub struct TaskIds {
 }
 
 impl TaskIds {
-    fn insert(&self, u: UniqueWeight) -> bool {
+    fn insert(&mut self, u: UniqueWeight) -> bool {
         self.task_ids.insert(u)
     }
 
-    fn remove(&self, u: &UniqueWeight) -> bool {
+    fn remove(&mut self, u: &UniqueWeight) -> bool {
         self.task_ids.remove(u)
     }
 

From 3c6b8eb7416e5dcc95225756400efb94ee45d1a4 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 13 Aug 2022 22:32:59 +0900
Subject: [PATCH 0608/3199] save

---
 scheduler/src/lib.rs | 1 +
 1 file changed, 1 insertion(+)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 9e9d5f7eeaded5..76d83860c09f3e 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -144,6 +144,7 @@ pub enum RequestedUsage {
 #[derive(Debug, Default)]
 pub struct TaskIds {
     task_ids: std::collections::BTreeSet<UniqueWeight>,
+    cached_heviest: Option<UniqueWeight>,
 }
 
 impl TaskIds {

From 0183849eaaeba4381a07a395d6237793fb4243c7 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 13 Aug 2022 22:34:43 +0900
Subject: [PATCH 0609/3199] save

---
 scheduler/src/lib.rs | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 76d83860c09f3e..54cbc7aa52b2e2 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -144,11 +144,17 @@ pub enum RequestedUsage {
 #[derive(Debug, Default)]
 pub struct TaskIds {
     task_ids: std::collections::BTreeSet<UniqueWeight>,
-    cached_heviest: Option<UniqueWeight>,
+    cached_heaviest: Option<UniqueWeight>,
 }
 
 impl TaskIds {
     fn insert(&mut self, u: UniqueWeight) -> bool {
+        match self.cached_heaviest {
+            Some(c) if u > c => { self.cached_heaviest = u },
+            None => { self.cached_heaviest = u; }
+            _,
+        }
+             
         self.task_ids.insert(u)
     }
 

From f54ecb8d0edc09037a1b9e7a69749ab9ab16d7f2 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 13 Aug 2022 22:34:57 +0900
Subject: [PATCH 0610/3199] save

---
 scheduler/src/lib.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 54cbc7aa52b2e2..a1e699d1d79f15 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -150,8 +150,8 @@ pub struct TaskIds {
 impl TaskIds {
     fn insert(&mut self, u: UniqueWeight) -> bool {
         match self.cached_heaviest {
-            Some(c) if u > c => { self.cached_heaviest = u },
-            None => { self.cached_heaviest = u; }
+            Some(c) if u > c => { self.cached_heaviest = Some(u) },
+            None => { self.cached_heaviest = Some(u); }
             _,
         }
              

From 27b8e4393ac723079c8bb99cf03275c24d7e8617 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 13 Aug 2022 22:35:07 +0900
Subject: [PATCH 0611/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index a1e699d1d79f15..4ee5117968a415 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -152,7 +152,7 @@ impl TaskIds {
         match self.cached_heaviest {
             Some(c) if u > c => { self.cached_heaviest = Some(u) },
             None => { self.cached_heaviest = Some(u); }
-            _,
+            _ => {},
         }
              
         self.task_ids.insert(u)

From a05c2d8510fe52ed5f611a1f22b1f7bdaec18810 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 13 Aug 2022 22:36:20 +0900
Subject: [PATCH 0612/3199] save

---
 scheduler/src/lib.rs | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 4ee5117968a415..6d5f1e99a5d205 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -159,7 +159,12 @@ impl TaskIds {
     }
 
     fn remove(&mut self, u: &UniqueWeight) -> bool {
-        self.task_ids.remove(u)
+        let a = self.task_ids.remove(u);
+        match self.cached_heaviest {
+            Some(c) if u > c => { self.cached_heaviest = self.task_ids.last() },
+            _ => {},
+        }
+        a
     }
 
     fn is_empty(&self) -> bool {

From 9c00524fc2f212cb0519699f39a9fd3f2a852782 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 13 Aug 2022 22:36:40 +0900
Subject: [PATCH 0613/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 6d5f1e99a5d205..fd7f6f43666aa1 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -161,7 +161,7 @@ impl TaskIds {
     fn remove(&mut self, u: &UniqueWeight) -> bool {
         let a = self.task_ids.remove(u);
         match self.cached_heaviest {
-            Some(c) if u > c => { self.cached_heaviest = self.task_ids.last() },
+            Some(ref c) if u > c => { self.cached_heaviest = self.task_ids.last() },
             _ => {},
         }
         a

From 8b0092762ecabe684401e40979d494793be47944 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 13 Aug 2022 22:36:59 +0900
Subject: [PATCH 0614/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index fd7f6f43666aa1..a3a073a4ab3322 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -161,7 +161,7 @@ impl TaskIds {
     fn remove(&mut self, u: &UniqueWeight) -> bool {
         let a = self.task_ids.remove(u);
         match self.cached_heaviest {
-            Some(ref c) if u > c => { self.cached_heaviest = self.task_ids.last() },
+            Some(ref c) if u > c => { self.cached_heaviest = self.task_ids.last().copied() },
             _ => {},
         }
         a

From a97a7e03b1d8e308eaaf28ac1d6aacc9c90974fa Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 13 Aug 2022 22:37:56 +0900
Subject: [PATCH 0615/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index a3a073a4ab3322..988056b87bc4b0 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -161,7 +161,7 @@ impl TaskIds {
     fn remove(&mut self, u: &UniqueWeight) -> bool {
         let a = self.task_ids.remove(u);
         match self.cached_heaviest {
-            Some(ref c) if u > c => { self.cached_heaviest = self.task_ids.last().copied() },
+            Some(ref c) if u == c => { self.cached_heaviest = self.task_ids.last().copied() },
             _ => {},
         }
         a

From 43749b3fb16a760b8ea7d3d680c1eb7d351af34b Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 13 Aug 2022 22:38:32 +0900
Subject: [PATCH 0616/3199] save

---
 scheduler/src/lib.rs | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 988056b87bc4b0..5ebd4cc1874b6a 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -172,7 +172,8 @@ impl TaskIds {
     }
 
     fn last(&self) -> Option<&UniqueWeight> {
-        self.task_ids.last()
+        //self.task_ids.last()
+        self.cached_heaviest.as_ref()
     }
 }
 

From 65da579c79d6a2f2c43c2f507a477d540655f4b0 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 13 Aug 2022 22:47:50 +0900
Subject: [PATCH 0617/3199] save

---
 ledger-tool/src/main.rs | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/ledger-tool/src/main.rs b/ledger-tool/src/main.rs
index 3a500b49dfba88..ac36a53585a05f 100644
--- a/ledger-tool/src/main.rs
+++ b/ledger-tool/src/main.rs
@@ -384,7 +384,8 @@ fn output_slot(
 
                 muxed_sender
                     .send(solana_scheduler::Multiplexed::FromPrevious((
-                        Weight { ix: weight },
+                        //Weight { ix: weight },
+                        weight,
                         tx,
                     )))
                     .unwrap();

From cb8683426ffcd7b1a6f842e0ae8b78ec859b7920 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 13 Aug 2022 22:50:36 +0900
Subject: [PATCH 0618/3199] save

---
 scheduler/src/lib.rs | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 5ebd4cc1874b6a..24b69963ea47d5 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -148,6 +148,7 @@ pub struct TaskIds {
 }
 
 impl TaskIds {
+    #[inline(never)]
     fn insert(&mut self, u: UniqueWeight) -> bool {
         match self.cached_heaviest {
             Some(c) if u > c => { self.cached_heaviest = Some(u) },
@@ -158,6 +159,7 @@ impl TaskIds {
         self.task_ids.insert(u)
     }
 
+    #[inline(never)]
     fn remove(&mut self, u: &UniqueWeight) -> bool {
         let a = self.task_ids.remove(u);
         match self.cached_heaviest {
@@ -167,10 +169,12 @@ impl TaskIds {
         a
     }
 
+    #[inline(never)]
     fn is_empty(&self) -> bool {
         self.task_ids.is_empty()
     }
 
+    #[inline(never)]
     fn last(&self) -> Option<&UniqueWeight> {
         //self.task_ids.last()
         self.cached_heaviest.as_ref()

From 276af406b98e85443cb77c7f3fd97f0a724b68a5 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 14 Aug 2022 21:40:19 +0900
Subject: [PATCH 0619/3199] save

---
 scheduler/src/lib.rs | 34 ++--------------------------------
 1 file changed, 2 insertions(+), 32 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 24b69963ea47d5..dc4c0c1da758fc 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -25,7 +25,7 @@ unsafe impl Send for PageRc {}
 type MyRcInner<T> = std::sync::Arc<T>;
 
 #[derive(Debug, Hash, PartialEq, Eq, Clone)]
-pub struct PageRc(ByAddress<MyRcInner<Page>>);
+pub struct PageRc(MyRcInner<Page>);
 
 
 #[derive(Debug)]
@@ -49,35 +49,6 @@ impl ExecutionEnvironment {
     //}
 }
 
-#[derive(Clone, Debug)]
-enum AddressLookup {
-    Before(Pubkey),
-    After(PageRc),
-}
-
-impl AddressLookup {
-    fn page_rc(&mut self) -> &PageRc {
-        match self {
-            AddressLookup::Before(_) => unreachable!(),
-            AddressLookup::After(page) => page,
-        }
-    }
-
-    fn take_page_rc(mut self) -> PageRc {
-        match self {
-            AddressLookup::Before(_) => unreachable!(),
-            AddressLookup::After(page) => page,
-        }
-    }
-
-    fn page(&mut self) -> &mut Page {
-        match self {
-            AddressLookup::Before(_) => unreachable!(),
-            AddressLookup::After(page) => unsafe { MyRcInner::get_mut_unchecked(&mut page.0) },
-        }
-    }
-}
-
 impl PageRc {
     fn page(&mut self) -> &mut Page {
         unsafe { MyRcInner::get_mut_unchecked(&mut self.0) }
@@ -210,7 +181,6 @@ impl Page {
 
 //type AddressMap = std::collections::HashMap<Pubkey, PageRc>;
 type AddressMap = std::sync::Arc<dashmap::DashMap<Pubkey, PageRc>>;
-use by_address::ByAddress;
 type TaskId = UniqueWeight;
 type WeightedTaskIds = std::collections::BTreeMap<TaskId, ()>;
 //type AddressMapEntry<'a, K, V> = std::collections::hash_map::Entry<'a, K, V>;
@@ -396,7 +366,7 @@ impl Preloader {
     pub fn load(&self, address: Pubkey) -> PageRc {
         match self.book.entry(address) {
             AddressMapEntry::Vacant(book_entry) => {
-                let page = PageRc(ByAddress(MyRcInner::new(Page::new(Usage::unused()))));
+                let page = PageRc(MyRcInner::new(Page::new(Usage::unused())));
                 let cloned = PageRc::clone(&page);
                 book_entry.insert(page);
                 cloned

From 5d8dc94bedb02eab3c0a9735631908a80d1cff0f Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 14 Aug 2022 21:41:25 +0900
Subject: [PATCH 0620/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index dc4c0c1da758fc..a02e1f0f1bcee5 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -24,7 +24,7 @@ unsafe impl Send for PageRc {}
 
 type MyRcInner<T> = std::sync::Arc<T>;
 
-#[derive(Debug, Hash, PartialEq, Eq, Clone)]
+#[derive(Debug, Clone)]
 pub struct PageRc(MyRcInner<Page>);
 
 

From d38a214ad9a8f88f59b7d8c580257c9206036f52 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 14 Aug 2022 21:45:51 +0900
Subject: [PATCH 0621/3199] save

---
 transaction-scheduler-bench/src/main.rs | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/transaction-scheduler-bench/src/main.rs b/transaction-scheduler-bench/src/main.rs
index 0a63a0df3da824..b95c83d31818ba 100644
--- a/transaction-scheduler-bench/src/main.rs
+++ b/transaction-scheduler-bench/src/main.rs
@@ -340,9 +340,10 @@ fn handle_transaction_batch(
         .priority_collected
         .fetch_add(priority_collected, Ordering::Relaxed);
 
-        completed_transaction_sender
-            .send(solana_scheduler::Multiplexed::FromExecute(transaction_batch))
-            .unwrap();
+    dbg!("send: {:?}", transaction_batch.unique_weight);
+    completed_transaction_sender
+        .send(solana_scheduler::Multiplexed::FromExecute(transaction_batch))
+        .unwrap();
 }
 
 const NUM_SENDERS: usize = 2;

From 2eec53632a277deb4ef3ec3dfc3f377227976b24 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 14 Aug 2022 21:47:12 +0900
Subject: [PATCH 0622/3199] save

---
 scheduler/src/lib.rs | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index a02e1f0f1bcee5..5d9cdf49bb8c5d 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -33,6 +33,7 @@ pub struct ExecutionEnvironment {
     lock_attempts: Vec<LockAttempt>,
     //accounts: Vec<i8>,
     pub cu: usize,
+    pub unique_weight: UniqueWeight,
     pub task: Task,
 }
 
@@ -806,6 +807,7 @@ impl ScheduleStage {
         Box::new(ExecutionEnvironment {
             lock_attempts,
             task,
+            unique_weight,
             cu: rng.gen_range(3, 1000),
         })
     }

From 4e339d33d31bd561e92be2cfefd7d1ead99b5912 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 14 Aug 2022 21:47:46 +0900
Subject: [PATCH 0623/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 5d9cdf49bb8c5d..7287b9fe7787fc 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -895,7 +895,7 @@ impl ScheduleStage {
                     }
                 }
                 Multiplexed::FromExecute(mut processed_execution_environment) => {
-                    trace!("recv from execute");
+                    trace!("recv from execute: {:?}", processed_execution_environment.unique_weight);
                     executing_queue_count -= 1;
 
                     Self::commit_result(&mut processed_execution_environment, address_book);

From 07a5caaea5a106ea5e37f8159b2a0f1ff7fae235 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 14 Aug 2022 21:48:09 +0900
Subject: [PATCH 0624/3199] save

---
 transaction-scheduler-bench/src/main.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/transaction-scheduler-bench/src/main.rs b/transaction-scheduler-bench/src/main.rs
index b95c83d31818ba..9647e2caf23f41 100644
--- a/transaction-scheduler-bench/src/main.rs
+++ b/transaction-scheduler-bench/src/main.rs
@@ -340,10 +340,10 @@ fn handle_transaction_batch(
         .priority_collected
         .fetch_add(priority_collected, Ordering::Relaxed);
 
-    dbg!("send: {:?}", transaction_batch.unique_weight);
     completed_transaction_sender
         .send(solana_scheduler::Multiplexed::FromExecute(transaction_batch))
         .unwrap();
+    dbg!("send: {:?}", transaction_batch.unique_weight);
 }
 
 const NUM_SENDERS: usize = 2;

From b609d86ee52a78236da61960d2de208c3e089244 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 14 Aug 2022 21:48:57 +0900
Subject: [PATCH 0625/3199] save

---
 transaction-scheduler-bench/src/main.rs | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/transaction-scheduler-bench/src/main.rs b/transaction-scheduler-bench/src/main.rs
index 9647e2caf23f41..bf9418391849b9 100644
--- a/transaction-scheduler-bench/src/main.rs
+++ b/transaction-scheduler-bench/src/main.rs
@@ -340,10 +340,11 @@ fn handle_transaction_batch(
         .priority_collected
         .fetch_add(priority_collected, Ordering::Relaxed);
 
+    let uq = transaction_batch.unique_weight;
     completed_transaction_sender
         .send(solana_scheduler::Multiplexed::FromExecute(transaction_batch))
         .unwrap();
-    dbg!("send: {:?}", transaction_batch.unique_weight);
+    dbg!("send: {:?}", uq);
 }
 
 const NUM_SENDERS: usize = 2;

From 9fd8f389cf36c795c66b9ffaa669f207e5a96a14 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 14 Aug 2022 21:50:01 +0900
Subject: [PATCH 0626/3199] save

---
 transaction-scheduler-bench/src/main.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/transaction-scheduler-bench/src/main.rs b/transaction-scheduler-bench/src/main.rs
index bf9418391849b9..165f06cfc7919f 100644
--- a/transaction-scheduler-bench/src/main.rs
+++ b/transaction-scheduler-bench/src/main.rs
@@ -344,7 +344,7 @@ fn handle_transaction_batch(
     completed_transaction_sender
         .send(solana_scheduler::Multiplexed::FromExecute(transaction_batch))
         .unwrap();
-    dbg!("send: {:?}", uq);
+    trace!("send from execute: {:?}", uq);
 }
 
 const NUM_SENDERS: usize = 2;

From 7d2e8d46b73a66ef7bf200cb9ec84c8f2a42acb4 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 14 Aug 2022 21:51:01 +0900
Subject: [PATCH 0627/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 7287b9fe7787fc..6f13f1c4fe6411 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -860,7 +860,7 @@ impl ScheduleStage {
         let mut current_unique_key = u64::max_value();
 
         loop {
-            trace!("schedule_once (runnnable: {}, contended: {}, (immediate+guaranteed)/max: ({}+{})/{})!", runnable_queue.task_count(), contended_queue.task_count(), executing_queue_count, address_book.gurantee_timers.len(), max_executing_queue_count);
+            trace!("schedule_once (from: {}, to: {}, runnnable: {}, contended: {}, (immediate+guaranteed)/max: ({}+{})/{})!", from.len(), to_execute_substage.len(), runnable_queue.task_count(), contended_queue.task_count(), executing_queue_count, address_book.gurantee_timers.len(), max_executing_queue_count);
 
             let i = from.recv().unwrap();
             match i {

From 85f121e61a21d85863a42eab5e1cde17aba465a4 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 14 Aug 2022 21:51:33 +0900
Subject: [PATCH 0628/3199] save

---
 transaction-scheduler-bench/src/main.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/transaction-scheduler-bench/src/main.rs b/transaction-scheduler-bench/src/main.rs
index 165f06cfc7919f..35d89d71327759 100644
--- a/transaction-scheduler-bench/src/main.rs
+++ b/transaction-scheduler-bench/src/main.rs
@@ -1,7 +1,7 @@
 use {
     clap::Parser,
     crossbeam_channel::{select, Receiver, Sender},
-    log::info,
+    log::*,
     rand::Rng,
     solana_core::{
         transaction_priority_details::GetTransactionPriorityDetails,

From 97ce4034f01f519171789caf39a82bffdff97b4f Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 14 Aug 2022 22:07:25 +0900
Subject: [PATCH 0629/3199] save

---
 scheduler/src/lib.rs | 15 ++++++++++++++-
 1 file changed, 14 insertions(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 6f13f1c4fe6411..4fc326267cad48 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -487,7 +487,7 @@ type PreprocessedTransaction = (SanitizedTransaction, Vec<LockAttempt>);
 pub enum Multiplexed {
     FromPrevious((Weight, Box<PreprocessedTransaction>)),
     FromPreviousBatched(Vec<Vec<Box<PreprocessedTransaction>>>),
-    FromExecute(Box<ExecutionEnvironment>),
+    HintFromExecute,
 }
 
 pub fn get_transaction_priority_details(tx: &SanitizedTransaction) -> u64 {
@@ -853,6 +853,7 @@ impl ScheduleStage {
         contended_queue: &mut TaskQueue,
         address_book: &mut AddressBook,
         from: &crossbeam_channel::Receiver<Multiplexed>,
+        from_exec: &crossbeam_channel::Receiver<Box<ExecutionEnvironment>>,
         to_execute_substage: &crossbeam_channel::Sender<Box<ExecutionEnvironment>>,
         to_next_stage: Option<&crossbeam_channel::Sender<Box<ExecutionEnvironment>>>, // assume nonblocking
     ) {
@@ -862,6 +863,18 @@ impl ScheduleStage {
         loop {
             trace!("schedule_once (from: {}, to: {}, runnnable: {}, contended: {}, (immediate+guaranteed)/max: ({}+{})/{})!", from.len(), to_execute_substage.len(), runnable_queue.task_count(), contended_queue.task_count(), executing_queue_count, address_book.gurantee_timers.len(), max_executing_queue_count);
 
+            if from_exec.len() > 0 {
+                let processed_execution_environment = from_exec.recv().unwrap();
+                trace!("recv from execute: {:?}", processed_execution_environment.unique_weight);
+                executing_queue_count -= 1;
+
+                Self::commit_result(&mut processed_execution_environment, address_book);
+                // async-ly propagate the result to rpc subsystems
+                if let Some(to_next_stage) = to_next_stage {
+                    to_next_stage.send(processed_execution_environment).unwrap();
+                }
+            }
+
             let i = from.recv().unwrap();
             match i {
                 Multiplexed::FromPrevious(weighted_tx) => {

From 1b31b09f65b3c894082056b122774d556c4ef07e Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 14 Aug 2022 22:08:24 +0900
Subject: [PATCH 0630/3199] save

---
 scheduler/src/lib.rs | 19 +++++++++++--------
 1 file changed, 11 insertions(+), 8 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 4fc326267cad48..8973643844ff4a 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -907,14 +907,17 @@ impl ScheduleStage {
                         }
                     }
                 }
-                Multiplexed::FromExecute(mut processed_execution_environment) => {
-                    trace!("recv from execute: {:?}", processed_execution_environment.unique_weight);
-                    executing_queue_count -= 1;
-
-                    Self::commit_result(&mut processed_execution_environment, address_book);
-                    // async-ly propagate the result to rpc subsystems
-                    if let Some(to_next_stage) = to_next_stage {
-                        to_next_stage.send(processed_execution_environment).unwrap();
+                Multiplexed::HintFromExecute => {
+                    if from_exec.len() > 0 {
+                        let processed_execution_environment = from_exec.recv().unwrap();
+                        trace!("recv from execute: {:?}", processed_execution_environment.unique_weight);
+                        executing_queue_count -= 1;
+
+                        Self::commit_result(&mut processed_execution_environment, address_book);
+                        // async-ly propagate the result to rpc subsystems
+                        if let Some(to_next_stage) = to_next_stage {
+                            to_next_stage.send(processed_execution_environment).unwrap();
+                        }
                     }
                 }
             }

From f0f88181138e79b3b5180ec9f3da11478a698042 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 14 Aug 2022 22:08:37 +0900
Subject: [PATCH 0631/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 8973643844ff4a..8268a79041be71 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -864,7 +864,7 @@ impl ScheduleStage {
             trace!("schedule_once (from: {}, to: {}, runnnable: {}, contended: {}, (immediate+guaranteed)/max: ({}+{})/{})!", from.len(), to_execute_substage.len(), runnable_queue.task_count(), contended_queue.task_count(), executing_queue_count, address_book.gurantee_timers.len(), max_executing_queue_count);
 
             if from_exec.len() > 0 {
-                let processed_execution_environment = from_exec.recv().unwrap();
+                let mut processed_execution_environment = from_exec.recv().unwrap();
                 trace!("recv from execute: {:?}", processed_execution_environment.unique_weight);
                 executing_queue_count -= 1;
 

From ae4eb0bfad440fee067609fd3c02a39c83a3f38e Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 14 Aug 2022 22:08:46 +0900
Subject: [PATCH 0632/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 8268a79041be71..24c2d7257a7d39 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -909,7 +909,7 @@ impl ScheduleStage {
                 }
                 Multiplexed::HintFromExecute => {
                     if from_exec.len() > 0 {
-                        let processed_execution_environment = from_exec.recv().unwrap();
+                        let mut processed_execution_environment = from_exec.recv().unwrap();
                         trace!("recv from execute: {:?}", processed_execution_environment.unique_weight);
                         executing_queue_count -= 1;
 

From df34950430177abf98c6773d454a57ae50ec9703 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 14 Aug 2022 22:10:50 +0900
Subject: [PATCH 0633/3199] save

---
 transaction-scheduler-bench/src/main.rs | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/transaction-scheduler-bench/src/main.rs b/transaction-scheduler-bench/src/main.rs
index 35d89d71327759..be6a5c46855fca 100644
--- a/transaction-scheduler-bench/src/main.rs
+++ b/transaction-scheduler-bench/src/main.rs
@@ -121,7 +121,7 @@ fn spawn_unified_scheduler(
         num_execution_threads: usize,
         packet_batch_receiver: Receiver<BatchSenderMessage>,
         transaction_batch_senders: Vec<Sender<TransactionBatchMessage>>,
-        //completed_transaction_receiver: Receiver<CompletedTransactionMessage>,
+        completed_transaction_receiver: Receiver<CompletedTransactionMessage>,
         bank_forks: Arc<RwLock<BankForks>>,
         max_batch_size: usize,
         exit: Arc<AtomicBool>,
@@ -136,6 +136,7 @@ fn spawn_unified_scheduler(
             &mut contended_queue,
             &mut address_book,
             &packet_batch_receiver.clone(),
+            completed_transaction_receiver,
             &transaction_batch_senders[0],
             None,//&completed_transaction_receiver
         );

From 8cfcecd19ede23fda98eb856b527713246947cd1 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 14 Aug 2022 22:13:03 +0900
Subject: [PATCH 0634/3199] save

---
 transaction-scheduler-bench/src/main.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/transaction-scheduler-bench/src/main.rs b/transaction-scheduler-bench/src/main.rs
index be6a5c46855fca..af19d06022cb9e 100644
--- a/transaction-scheduler-bench/src/main.rs
+++ b/transaction-scheduler-bench/src/main.rs
@@ -187,7 +187,7 @@ fn main() {
     let execution_handles = start_execution_threads(
         metrics.clone(),
         transaction_batch_receivers,
-        packet_batch_sender.clone(),
+        (completed_transaction_sender.clone(), packet_batch_sender.clone()),
         execution_per_tx_us,
         exit.clone(),
     );

From 01a642ceba7b3d15e24757ef1efdb9cb460cbd3a Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 14 Aug 2022 22:16:22 +0900
Subject: [PATCH 0635/3199] save

---
 transaction-scheduler-bench/src/main.rs | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/transaction-scheduler-bench/src/main.rs b/transaction-scheduler-bench/src/main.rs
index af19d06022cb9e..d7a984cecde453 100644
--- a/transaction-scheduler-bench/src/main.rs
+++ b/transaction-scheduler-bench/src/main.rs
@@ -165,6 +165,7 @@ fn main() {
     assert!(high_conflict_sender <= num_accounts);
 
     let (packet_batch_sender, packet_batch_receiver) = crossbeam_channel::unbounded();
+    let (completed_transaction_sender, completed_transaction_receiver) = crossbeam_channel::unbounded();
     let (transaction_batch_senders, transaction_batch_receivers) =
         build_channels(num_execution_threads);
     let bank_forks = Arc::new(RwLock::new(BankForks::new(Bank::default_for_tests())));
@@ -176,6 +177,7 @@ fn main() {
         num_execution_threads,
         packet_batch_receiver,
         transaction_batch_senders,
+        completed_transaction_receiver,
         bank_forks,
         max_batch_size,
         exit.clone(),
@@ -187,7 +189,7 @@ fn main() {
     let execution_handles = start_execution_threads(
         metrics.clone(),
         transaction_batch_receivers,
-        (completed_transaction_sender.clone(), packet_batch_sender.clone()),
+        (completed_transaction_sender, packet_batch_sender.clone()),
         execution_per_tx_us,
         exit.clone(),
     );

From 7486ce5719bf806e53f8f377ccd414df6d841d57 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 14 Aug 2022 22:17:17 +0900
Subject: [PATCH 0636/3199] save

---
 transaction-scheduler-bench/src/main.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/transaction-scheduler-bench/src/main.rs b/transaction-scheduler-bench/src/main.rs
index d7a984cecde453..dcf90f972a9751 100644
--- a/transaction-scheduler-bench/src/main.rs
+++ b/transaction-scheduler-bench/src/main.rs
@@ -136,7 +136,7 @@ fn spawn_unified_scheduler(
             &mut contended_queue,
             &mut address_book,
             &packet_batch_receiver.clone(),
-            completed_transaction_receiver,
+            &completed_transaction_receiver,
             &transaction_batch_senders[0],
             None,//&completed_transaction_receiver
         );

From 934b52e3aadf88908aa15087e01142238bb2678a Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 14 Aug 2022 22:19:20 +0900
Subject: [PATCH 0637/3199] save

---
 transaction-scheduler-bench/src/main.rs | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/transaction-scheduler-bench/src/main.rs b/transaction-scheduler-bench/src/main.rs
index dcf90f972a9751..6ca88055a033b1 100644
--- a/transaction-scheduler-bench/src/main.rs
+++ b/transaction-scheduler-bench/src/main.rs
@@ -80,7 +80,8 @@ struct Args {
 /// Some convenient type aliases
 type PreprocessedTransaction = Box<(SanitizedTransaction, Vec<solana_scheduler::LockAttempt>)>;
 type TransactionMessage = PreprocessedTransaction;
-type CompletedTransactionMessage = solana_scheduler::Multiplexed; // (usize, Box<solana_scheduler::ExecutionEnvironment>); //(usize, TransactionMessage); // thread index and transaction message
+//type CompletedTransactionMessage = solana_scheduler::Multiplexed; // (usize, Box<solana_scheduler::ExecutionEnvironment>); //(usize, TransactionMessage); // thread index and transaction message
+type CompletedTransactionMessage = Box<solana_scheduler::ExecutionEnvironment>);
 type TransactionBatchMessage = Box<solana_scheduler::ExecutionEnvironment>; // Vec<TransactionMessage>;
 type BatchSenderMessage = solana_scheduler::Multiplexed; // Vec<Vec<PreprocessedTransaction>>;
 

From 9092a5f1f6ae0a182bad58d50965e50e3f7198b9 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 14 Aug 2022 22:19:27 +0900
Subject: [PATCH 0638/3199] save

---
 transaction-scheduler-bench/src/main.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/transaction-scheduler-bench/src/main.rs b/transaction-scheduler-bench/src/main.rs
index 6ca88055a033b1..8ef4d2585ff357 100644
--- a/transaction-scheduler-bench/src/main.rs
+++ b/transaction-scheduler-bench/src/main.rs
@@ -81,7 +81,7 @@ struct Args {
 type PreprocessedTransaction = Box<(SanitizedTransaction, Vec<solana_scheduler::LockAttempt>)>;
 type TransactionMessage = PreprocessedTransaction;
 //type CompletedTransactionMessage = solana_scheduler::Multiplexed; // (usize, Box<solana_scheduler::ExecutionEnvironment>); //(usize, TransactionMessage); // thread index and transaction message
-type CompletedTransactionMessage = Box<solana_scheduler::ExecutionEnvironment>);
+type CompletedTransactionMessage = Box<solana_scheduler::ExecutionEnvironment>;
 type TransactionBatchMessage = Box<solana_scheduler::ExecutionEnvironment>; // Vec<TransactionMessage>;
 type BatchSenderMessage = solana_scheduler::Multiplexed; // Vec<Vec<PreprocessedTransaction>>;
 

From 3af6b816eb8e28dac0e1920f8dc5d184b337a2e9 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 14 Aug 2022 22:20:08 +0900
Subject: [PATCH 0639/3199] save

---
 transaction-scheduler-bench/src/main.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/transaction-scheduler-bench/src/main.rs b/transaction-scheduler-bench/src/main.rs
index 8ef4d2585ff357..99d7cc58a19b80 100644
--- a/transaction-scheduler-bench/src/main.rs
+++ b/transaction-scheduler-bench/src/main.rs
@@ -255,7 +255,7 @@ fn main() {
 fn start_execution_threads(
     metrics: Arc<TransactionSchedulerBenchMetrics>,
     transaction_batch_receivers: Vec<Receiver<TransactionBatchMessage>>,
-    completed_transaction_sender: Sender<CompletedTransactionMessage>,
+    completed_transaction_sender: (Sender<CompletedTransactionMessage>, Sender<Multiplexed>),
     execution_per_tx_us: u64,
     exit: Arc<AtomicBool>,
 ) -> Vec<JoinHandle<()>> {

From 57a32a9f94417e002338ae639ae5edcdc723ab5f Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 14 Aug 2022 22:20:18 +0900
Subject: [PATCH 0640/3199] save

---
 transaction-scheduler-bench/src/main.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/transaction-scheduler-bench/src/main.rs b/transaction-scheduler-bench/src/main.rs
index 99d7cc58a19b80..9967ddadd501b8 100644
--- a/transaction-scheduler-bench/src/main.rs
+++ b/transaction-scheduler-bench/src/main.rs
@@ -255,7 +255,7 @@ fn main() {
 fn start_execution_threads(
     metrics: Arc<TransactionSchedulerBenchMetrics>,
     transaction_batch_receivers: Vec<Receiver<TransactionBatchMessage>>,
-    completed_transaction_sender: (Sender<CompletedTransactionMessage>, Sender<Multiplexed>),
+    completed_transaction_sender: (Sender<CompletedTransactionMessage>, Sender<solana_scheduler::Multiplexed>),
     execution_per_tx_us: u64,
     exit: Arc<AtomicBool>,
 ) -> Vec<JoinHandle<()>> {

From 3c662c9d955abca2e63396489f01a00a9ab39be9 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 14 Aug 2022 22:20:46 +0900
Subject: [PATCH 0641/3199] save

---
 transaction-scheduler-bench/src/main.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/transaction-scheduler-bench/src/main.rs b/transaction-scheduler-bench/src/main.rs
index 9967ddadd501b8..660d17eabc5cf7 100644
--- a/transaction-scheduler-bench/src/main.rs
+++ b/transaction-scheduler-bench/src/main.rs
@@ -279,7 +279,7 @@ fn start_execution_thread(
     metrics: Arc<TransactionSchedulerBenchMetrics>,
     thread_index: usize,
     transaction_batch_receiver: Receiver<TransactionBatchMessage>,
-    completed_transaction_sender: Sender<CompletedTransactionMessage>,
+    completed_transaction_sender: (Sender<CompletedTransactionMessage>, Sender<solana_scheduler::Multiplexed>),
     execution_per_tx_us: u64,
     exit: Arc<AtomicBool>,
 ) -> JoinHandle<()> {

From bbf4609c457838fc04b85702cbc195b8ab1eb075 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 14 Aug 2022 22:21:09 +0900
Subject: [PATCH 0642/3199] save

---
 transaction-scheduler-bench/src/main.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/transaction-scheduler-bench/src/main.rs b/transaction-scheduler-bench/src/main.rs
index 660d17eabc5cf7..37940065adb265 100644
--- a/transaction-scheduler-bench/src/main.rs
+++ b/transaction-scheduler-bench/src/main.rs
@@ -299,7 +299,7 @@ fn execution_worker(
     metrics: Arc<TransactionSchedulerBenchMetrics>,
     thread_index: usize,
     transaction_batch_receiver: Receiver<TransactionBatchMessage>,
-    completed_transaction_sender: Sender<CompletedTransactionMessage>,
+    completed_transaction_sender: (Sender<CompletedTransactionMessage>, Sender<solana_scheduler::Multiplexed>),
     execution_per_tx_us: u64,
     exit: Arc<AtomicBool>,
 ) {

From 70d6f94dc70ad595854ab8c1045053fc00a3d06f Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 14 Aug 2022 22:21:43 +0900
Subject: [PATCH 0643/3199] save

---
 transaction-scheduler-bench/src/main.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/transaction-scheduler-bench/src/main.rs b/transaction-scheduler-bench/src/main.rs
index 37940065adb265..5b9008ca10aeb9 100644
--- a/transaction-scheduler-bench/src/main.rs
+++ b/transaction-scheduler-bench/src/main.rs
@@ -322,7 +322,7 @@ fn execution_worker(
 fn handle_transaction_batch(
     metrics: &TransactionSchedulerBenchMetrics,
     thread_index: usize,
-    completed_transaction_sender: &Sender<CompletedTransactionMessage>,
+    completed_transaction_sender: &(Sender<CompletedTransactionMessage>, Sender<solana_scheduler::Multiplexed>),
     transaction_batch: TransactionBatchMessage,
     execution_per_tx_us: u64,
 ) {

From d705e27306400dcd1a5ad0476b83e1f801f5ae38 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 14 Aug 2022 22:22:06 +0900
Subject: [PATCH 0644/3199] save

---
 transaction-scheduler-bench/src/main.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/transaction-scheduler-bench/src/main.rs b/transaction-scheduler-bench/src/main.rs
index 5b9008ca10aeb9..b4c7f6db3e1473 100644
--- a/transaction-scheduler-bench/src/main.rs
+++ b/transaction-scheduler-bench/src/main.rs
@@ -345,7 +345,7 @@ fn handle_transaction_batch(
         .fetch_add(priority_collected, Ordering::Relaxed);
 
     let uq = transaction_batch.unique_weight;
-    completed_transaction_sender
+    completed_transaction_sender.0
         .send(solana_scheduler::Multiplexed::FromExecute(transaction_batch))
         .unwrap();
     trace!("send from execute: {:?}", uq);

From e4a19bf3c4850f6a2292921638fdc4da606c807a Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 14 Aug 2022 22:22:36 +0900
Subject: [PATCH 0645/3199] save

---
 transaction-scheduler-bench/src/main.rs | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/transaction-scheduler-bench/src/main.rs b/transaction-scheduler-bench/src/main.rs
index b4c7f6db3e1473..3006cb2408c063 100644
--- a/transaction-scheduler-bench/src/main.rs
+++ b/transaction-scheduler-bench/src/main.rs
@@ -346,7 +346,10 @@ fn handle_transaction_batch(
 
     let uq = transaction_batch.unique_weight;
     completed_transaction_sender.0
-        .send(solana_scheduler::Multiplexed::FromExecute(transaction_batch))
+        .send(transaction_batch)
+        .unwrap();
+    completed_transaction_sender.1
+        .send(solana_scheduler::Multiplexed::HintFromExecute)
         .unwrap();
     trace!("send from execute: {:?}", uq);
 }

From 9441cfa36551fd8ea5a44b5e23695137bd83c4a6 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 14 Aug 2022 22:31:18 +0900
Subject: [PATCH 0646/3199] save

---
 scheduler/src/lib.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 24c2d7257a7d39..74058e3c42668e 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -586,10 +586,10 @@ impl ScheduleStage {
                 let weight_from_runnable = heaviest_runnable_entry.key();
                 let uw = weight_from_contended.key();
 
-                if true || weight_from_runnable > uw {
+                if weight_from_runnable > uw {
                     trace!("select: runnable > contended");
                     Some((Some(contended_queue), heaviest_runnable_entry))
-                } else if false && uw > weight_from_runnable {
+                } else if uw > weight_from_runnable {
                     trace!("select: contended > runnnable");
                     let uw = *uw;
                     weight_from_contended.remove();

From 70ac496cdc1afa5b0322430d372e850d31a01402 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 14 Aug 2022 22:33:51 +0900
Subject: [PATCH 0647/3199] save

---
 scheduler/src/lib.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 74058e3c42668e..886ea5200b2284 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -586,10 +586,10 @@ impl ScheduleStage {
                 let weight_from_runnable = heaviest_runnable_entry.key();
                 let uw = weight_from_contended.key();
 
-                if weight_from_runnable > uw {
+                if false && weight_from_runnable > uw {
                     trace!("select: runnable > contended");
                     Some((Some(contended_queue), heaviest_runnable_entry))
-                } else if uw > weight_from_runnable {
+                } else if true || uw > weight_from_runnable {
                     trace!("select: contended > runnnable");
                     let uw = *uw;
                     weight_from_contended.remove();

From de93997b9afd52da2acedf596d75ff6cb8e6e40f Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 14 Aug 2022 22:49:26 +0900
Subject: [PATCH 0648/3199] save

---
 scheduler/src/lib.rs | 12 +++++++-----
 1 file changed, 7 insertions(+), 5 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 886ea5200b2284..2cd3102bc6b063 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -680,8 +680,8 @@ impl ScheduleStage {
                 );
                 std::mem::swap(&mut next_task.tx.1, &mut populated_lock_attempts);
 
-                return None;
-                //continue;
+                //return None;
+                continue;
             }
 
             trace!("successful lock: (from_runnable: {}) after {} contentions", from_runnable, next_task.contention_count);
@@ -776,11 +776,13 @@ impl ScheduleStage {
                     match address_book.gurantee_timers.entry(*task_id) {
                         std::collections::hash_map::Entry::Occupied(mut entry) => {
                             let count = entry.get_mut();
-                            trace!("guaranteed lock decrease: {} => {}", *count, *count -1);
                             *count = count.checked_sub(1).unwrap();
                             if *count == 0 {
+                                trace!("guaranteed lock decrease: {} => {} (!)", *count + 1, *count);
                                 entry.remove();
                                 address_book.runnable_guaranteed_task_ids.insert(*task_id, ());
+                            } else {
+                                trace!("guaranteed lock decrease: {} => {}", *count + 1, *count);
                             }
                         },
                         std::collections::hash_map::Entry::Vacant(_) => {
@@ -861,7 +863,7 @@ impl ScheduleStage {
         let mut current_unique_key = u64::max_value();
 
         loop {
-            trace!("schedule_once (from: {}, to: {}, runnnable: {}, contended: {}, (immediate+guaranteed)/max: ({}+{})/{})!", from.len(), to_execute_substage.len(), runnable_queue.task_count(), contended_queue.task_count(), executing_queue_count, address_book.gurantee_timers.len(), max_executing_queue_count);
+            trace!("schedule_once (from: {}, to: {}, runnnable: {}, contended: {}, (immediate+guaranteed)/max: ({}+{})/{}) active from contended: {}!", from.len(), to_execute_substage.len(), runnable_queue.task_count(), contended_queue.task_count(), executing_queue_count, address_book.gurantee_timers.len(), max_executing_queue_count, address_book.uncontended_task_ids.len());
 
             if from_exec.len() > 0 {
                 let mut processed_execution_environment = from_exec.recv().unwrap();
@@ -939,7 +941,7 @@ impl ScheduleStage {
 
                     to_execute_substage.send(ee).unwrap();
                 } else {
-                    trace!("incoming queue starved: n_u_a: {}", address_book.uncontended_task_ids.len());
+                    trace!("incoming queue starved");
                     break;
                 }
             }

From 7617683674eda94281f516e28d7111a017896fc4 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 14 Aug 2022 22:50:45 +0900
Subject: [PATCH 0649/3199] save

---
 scheduler/src/lib.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 2cd3102bc6b063..6da7ec10f0b12b 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -680,8 +680,8 @@ impl ScheduleStage {
                 );
                 std::mem::swap(&mut next_task.tx.1, &mut populated_lock_attempts);
 
-                //return None;
-                continue;
+                return None;
+                //continue;
             }
 
             trace!("successful lock: (from_runnable: {}) after {} contentions", from_runnable, next_task.contention_count);

From bd8d6b90a333749d294404b570ed4fd2dca84ba9 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 15 Aug 2022 09:53:28 +0900
Subject: [PATCH 0650/3199] save

---
 scheduler/src/lib.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 6da7ec10f0b12b..07588ac9ce56d0 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -925,9 +925,9 @@ impl ScheduleStage {
             }
 
             loop {
-                /*if !address_book.uncontended_task_ids.is_empty() {
+                if !address_book.uncontended_task_ids.is_empty() {
                     trace!("prefer emptying n_u_a");
-                } else */ if (executing_queue_count + address_book.gurantee_timers.len()) >= max_executing_queue_count {
+                } else if (executing_queue_count + address_book.gurantee_timers.len()) >= max_executing_queue_count {
                     trace!("outgoing queue full");
                     break;
                 }

From 075b30130ac7641c025dd2852df5fe34e8ec233f Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 15 Aug 2022 10:03:25 +0900
Subject: [PATCH 0651/3199] save

---
 scheduler/src/lib.rs | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 07588ac9ce56d0..2e68c9a76798b9 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -910,6 +910,7 @@ impl ScheduleStage {
                     }
                 }
                 Multiplexed::HintFromExecute => {
+                    trace!("hint of recv from execute");
                     if from_exec.len() > 0 {
                         let mut processed_execution_environment = from_exec.recv().unwrap();
                         trace!("recv from execute: {:?}", processed_execution_environment.unique_weight);
@@ -928,7 +929,7 @@ impl ScheduleStage {
                 if !address_book.uncontended_task_ids.is_empty() {
                     trace!("prefer emptying n_u_a");
                 } else if (executing_queue_count + address_book.gurantee_timers.len()) >= max_executing_queue_count {
-                    trace!("outgoing queue full");
+                    trace!("skip scheduling; outgoing queue full");
                     break;
                 }
 

From 21dde940c479f4c38bf96370a9b328f3555e5e54 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 15 Aug 2022 10:08:40 +0900
Subject: [PATCH 0652/3199] save

---
 scheduler/src/lib.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 2e68c9a76798b9..c328615f19e7ad 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -586,10 +586,10 @@ impl ScheduleStage {
                 let weight_from_runnable = heaviest_runnable_entry.key();
                 let uw = weight_from_contended.key();
 
-                if false && weight_from_runnable > uw {
+                if weight_from_runnable > uw {
                     trace!("select: runnable > contended");
                     Some((Some(contended_queue), heaviest_runnable_entry))
-                } else if true || uw > weight_from_runnable {
+                } else uw > weight_from_runnable {
                     trace!("select: contended > runnnable");
                     let uw = *uw;
                     weight_from_contended.remove();

From a3488d869de0f8aad88ed10187e1bb2e75673228 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 15 Aug 2022 10:09:03 +0900
Subject: [PATCH 0653/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index c328615f19e7ad..306d91dafbd4aa 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -589,7 +589,7 @@ impl ScheduleStage {
                 if weight_from_runnable > uw {
                     trace!("select: runnable > contended");
                     Some((Some(contended_queue), heaviest_runnable_entry))
-                } else uw > weight_from_runnable {
+                } else if uw > weight_from_runnable {
                     trace!("select: contended > runnnable");
                     let uw = *uw;
                     weight_from_contended.remove();

From ff892f1e7eeb69803e89f030cfea842e13b8cf1d Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 15 Aug 2022 10:55:21 +0900
Subject: [PATCH 0654/3199] save

---
 scheduler/src/lib.rs | 16 +++++++++++-----
 1 file changed, 11 insertions(+), 5 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 306d91dafbd4aa..96dc896a421fe9 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -200,6 +200,7 @@ impl AddressBook {
     #[inline(never)]
     fn attempt_lock_address(
         from_runnable: bool,
+        prefer_immediate: bool,
         unique_weight: &UniqueWeight,
         attempt: &mut LockAttempt,
     ) {
@@ -227,7 +228,7 @@ impl AddressBook {
                             }
                         }
                         RequestedUsage::Writable => {
-                            if from_runnable {
+                            if from_runnable || prefer_immediate {
                                 //Self::remember_address_contention(&mut page, unique_weight);
                                 //*remembered = true;
                                 *status = LockStatus::Failed;
@@ -248,7 +249,7 @@ impl AddressBook {
                         }
                     },
                     Usage::Writable => {
-                        if from_runnable {
+                        if from_runnable || prefer_immediate {
                             //Self::remember_address_contention(&mut page, unique_weight);
                             //*remembered = true;
                             *status = LockStatus::Failed;
@@ -455,6 +456,7 @@ impl TaskQueue {
 #[inline(never)]
 fn attempt_lock_for_execution<'a>(
     from_runnable: bool,
+    prefer_immediate: bool,
     address_book: &mut AddressBook,
     unique_weight: &UniqueWeight,
     message_hash: &'a Hash,
@@ -465,7 +467,7 @@ fn attempt_lock_for_execution<'a>(
     let mut guaranteed_count = 0;
 
     for attempt in placeholder_attempts.iter_mut() {
-        AddressBook::attempt_lock_address(from_runnable, unique_weight, attempt);
+        AddressBook::attempt_lock_address(from_runnable, prefer_immediate, unique_weight, attempt);
         match attempt.status {
             LockStatus::Succeded => {},
             LockStatus::Failed => {
@@ -615,6 +617,7 @@ impl ScheduleStage {
         runnable_queue: &mut TaskQueue,
         contended_queue: &mut TaskQueue,
         address_book: &mut AddressBook,
+        prefer_immediate: bool,
     ) -> Option<(UniqueWeight, Task, Vec<LockAttempt>)> {
         if let Some(a) = address_book.runnable_guaranteed_task_ids.pop_last() {
             trace!("expediate pop from guaranteed queue [rest: {}]", address_book.runnable_guaranteed_task_ids.len());
@@ -639,6 +642,7 @@ impl ScheduleStage {
 
             let (unlockable_count, guaranteed_count, mut populated_lock_attempts) = attempt_lock_for_execution(
                 from_runnable,
+                prefer_immediate,
                 address_book,
                 &unique_weight,
                 &message_hash,
@@ -833,9 +837,10 @@ impl ScheduleStage {
         runnable_queue: &mut TaskQueue,
         contended_queue: &mut TaskQueue,
         address_book: &mut AddressBook,
+        prefer_immediate: bool,
     ) -> Option<Box<ExecutionEnvironment>> {
         let maybe_ee =
-            Self::pop_from_queue_then_lock(runnable_queue, contended_queue, address_book)
+            Self::pop_from_queue_then_lock(runnable_queue, contended_queue, address_book, prefer_immediate)
                 .map(|(uw, t, ll)| Self::prepare_scheduled_execution(address_book, uw, t, ll));
         maybe_ee
     }
@@ -933,8 +938,9 @@ impl ScheduleStage {
                     break;
                 }
 
+                let prefer_immediate = executing_queue_count < max_executing_queue_count / 2;
                 let maybe_ee =
-                    Self::schedule_next_execution(runnable_queue, contended_queue, address_book);
+                    Self::schedule_next_execution(runnable_queue, contended_queue, address_book, prefer_immediate);
 
                 if let Some(ee) = maybe_ee {
                     trace!("send to execute");

From 8aae41b76f53ee073a4b8d1fcbc77487848af8c3 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 15 Aug 2022 11:17:55 +0900
Subject: [PATCH 0655/3199] save

---
 scheduler/src/lib.rs | 76 ++++++++++++++++++--------------------------
 1 file changed, 31 insertions(+), 45 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 96dc896a421fe9..996b7cd13d7827 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -870,54 +870,35 @@ impl ScheduleStage {
         loop {
             trace!("schedule_once (from: {}, to: {}, runnnable: {}, contended: {}, (immediate+guaranteed)/max: ({}+{})/{}) active from contended: {}!", from.len(), to_execute_substage.len(), runnable_queue.task_count(), contended_queue.task_count(), executing_queue_count, address_book.gurantee_timers.len(), max_executing_queue_count, address_book.uncontended_task_ids.len());
 
-            if from_exec.len() > 0 {
-                let mut processed_execution_environment = from_exec.recv().unwrap();
-                trace!("recv from execute: {:?}", processed_execution_environment.unique_weight);
-                executing_queue_count -= 1;
-
-                Self::commit_result(&mut processed_execution_environment, address_book);
-                // async-ly propagate the result to rpc subsystems
-                if let Some(to_next_stage) = to_next_stage {
-                    to_next_stage.send(processed_execution_environment).unwrap();
-                }
-            }
-
-            let i = from.recv().unwrap();
-            match i {
-                Multiplexed::FromPrevious(weighted_tx) => {
-                    trace!("recv from previous");
-
-                    Self::register_runnable_task(weighted_tx, runnable_queue, &mut current_unique_key);
-                }
-                Multiplexed::FromPreviousBatched(vvv) => {
-                    trace!("recv from previous");
-
-                    for vv in vvv {
-                        for v in vv {
-                            panic!();
-                            //let p = get_transaction_priority_details(&v.0);
-                            //Self::register_runnable_task((Weight { ix: p }, v), runnable_queue, &mut current_unique_key);
-                            /*
-                            if executing_queue_count < max_executing_queue_count {
-                                let maybe_ee =
-                                    Self::schedule_next_execution(runnable_queue, contended_queue, address_book);
-                                if let Some(ee) = maybe_ee {
-                                    trace!("batched: send to execute");
-                                    executing_queue_count += 1;
-
-                                    to_execute_substage.send(ee).unwrap();
+            select! {
+               recv(from) => maybe_from => {
+                   let i = maybe_from.unwrap();
+                    match i {
+                        Multiplexed::FromPrevious(weighted_tx) => {
+                            trace!("recv from previous");
+
+                            while from_exec.len() > 0 {
+                                let mut processed_execution_environment = from_exec.recv().unwrap();
+                                trace!("recv from execute: {:?}", processed_execution_environment.unique_weight);
+                                executing_queue_count -= 1;
+
+                                Self::commit_result(&mut processed_execution_environment, address_book);
+                                // async-ly propagate the result to rpc subsystems
+                                if let Some(to_next_stage) = to_next_stage {
+                                    to_next_stage.send(processed_execution_environment).unwrap();
                                 }
-                            } else {
-                                    trace!("batched: outgoing queue full");
                             }
-                            */
+
+                            Self::register_runnable_task(weighted_tx, runnable_queue, &mut current_unique_key);
+                        }
+                        Multiplexed::FromPreviousBatched(vvv) => {
+                            unreachable!();
                         }
                     }
-                }
-                Multiplexed::HintFromExecute => {
-                    trace!("hint of recv from execute");
-                    if from_exec.len() > 0 {
-                        let mut processed_execution_environment = from_exec.recv().unwrap();
+               }
+               recv(from_exec) => maybe_from_exec => {
+                   let mut processed_execution_environment = maybe_from_exec.unwrap();
+                    loop {
                         trace!("recv from execute: {:?}", processed_execution_environment.unique_weight);
                         executing_queue_count -= 1;
 
@@ -926,8 +907,13 @@ impl ScheduleStage {
                         if let Some(to_next_stage) = to_next_stage {
                             to_next_stage.send(processed_execution_environment).unwrap();
                         }
+                        if from_exec.len() > 0 {
+                            processed_execution_environment = from_exec.recv().unwrap();
+                        } else {
+                            break;
+                        }
                     }
-                }
+               }
             }
 
             loop {

From 0d37ad3291704d12bfba9b6a194453063daac2ca Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 15 Aug 2022 11:18:17 +0900
Subject: [PATCH 0656/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 996b7cd13d7827..c95e341947329e 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -870,7 +870,7 @@ impl ScheduleStage {
         loop {
             trace!("schedule_once (from: {}, to: {}, runnnable: {}, contended: {}, (immediate+guaranteed)/max: ({}+{})/{}) active from contended: {}!", from.len(), to_execute_substage.len(), runnable_queue.task_count(), contended_queue.task_count(), executing_queue_count, address_book.gurantee_timers.len(), max_executing_queue_count, address_book.uncontended_task_ids.len());
 
-            select! {
+            crossbeam_channel::select! {
                recv(from) => maybe_from => {
                    let i = maybe_from.unwrap();
                     match i {

From a13684f7ebc7641bce91efb8b64ccc2321a02e1a Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 15 Aug 2022 11:19:09 +0900
Subject: [PATCH 0657/3199] save

---
 scheduler/src/lib.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index c95e341947329e..f8265e82c162df 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -871,7 +871,7 @@ impl ScheduleStage {
             trace!("schedule_once (from: {}, to: {}, runnnable: {}, contended: {}, (immediate+guaranteed)/max: ({}+{})/{}) active from contended: {}!", from.len(), to_execute_substage.len(), runnable_queue.task_count(), contended_queue.task_count(), executing_queue_count, address_book.gurantee_timers.len(), max_executing_queue_count, address_book.uncontended_task_ids.len());
 
             crossbeam_channel::select! {
-               recv(from) => maybe_from => {
+               recv(from) -> maybe_from => {
                    let i = maybe_from.unwrap();
                     match i {
                         Multiplexed::FromPrevious(weighted_tx) => {
@@ -896,7 +896,7 @@ impl ScheduleStage {
                         }
                     }
                }
-               recv(from_exec) => maybe_from_exec => {
+               recv(from_exec) -> maybe_from_exec => {
                    let mut processed_execution_environment = maybe_from_exec.unwrap();
                     loop {
                         trace!("recv from execute: {:?}", processed_execution_environment.unique_weight);

From d3f5e8019a7f479605d43763c1035f02bd5ec2e0 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 15 Aug 2022 11:19:29 +0900
Subject: [PATCH 0658/3199] save

---
 scheduler/src/lib.rs                    | 1 -
 transaction-scheduler-bench/src/main.rs | 3 ---
 2 files changed, 4 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index f8265e82c162df..47d2e36220ad64 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -489,7 +489,6 @@ type PreprocessedTransaction = (SanitizedTransaction, Vec<LockAttempt>);
 pub enum Multiplexed {
     FromPrevious((Weight, Box<PreprocessedTransaction>)),
     FromPreviousBatched(Vec<Vec<Box<PreprocessedTransaction>>>),
-    HintFromExecute,
 }
 
 pub fn get_transaction_priority_details(tx: &SanitizedTransaction) -> u64 {
diff --git a/transaction-scheduler-bench/src/main.rs b/transaction-scheduler-bench/src/main.rs
index 3006cb2408c063..52e82a30e9e973 100644
--- a/transaction-scheduler-bench/src/main.rs
+++ b/transaction-scheduler-bench/src/main.rs
@@ -348,9 +348,6 @@ fn handle_transaction_batch(
     completed_transaction_sender.0
         .send(transaction_batch)
         .unwrap();
-    completed_transaction_sender.1
-        .send(solana_scheduler::Multiplexed::HintFromExecute)
-        .unwrap();
     trace!("send from execute: {:?}", uq);
 }
 

From 709e4e9d9f7416b7cfdff295fa022aa3fb3c1b99 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 15 Aug 2022 11:22:18 +0900
Subject: [PATCH 0659/3199] save

---
 scheduler/src/lib.rs | 1 -
 1 file changed, 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 47d2e36220ad64..edce2cc9561ed8 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -555,7 +555,6 @@ impl ScheduleStage {
 
     #[inline(never)]
     fn get_heaviest_from_contended<'a>(address_book: &'a mut AddressBook) -> Option<std::collections::btree_map::OccupiedEntry<'a, UniqueWeight, ()>> {
-        trace!("n_u_a len(): {}", address_book.uncontended_task_ids.len());
         address_book.uncontended_task_ids.last_entry()
     }
 

From 040a4513b41da6950937af3d8fe9cc30e2a92a28 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 15 Aug 2022 11:32:05 +0900
Subject: [PATCH 0660/3199] save

---
 ledger-tool/src/main.rs | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/ledger-tool/src/main.rs b/ledger-tool/src/main.rs
index ac36a53585a05f..0a65f751acb351 100644
--- a/ledger-tool/src/main.rs
+++ b/ledger-tool/src/main.rs
@@ -265,7 +265,7 @@ fn output_slot(
     let (pre_execute_env_sender, pre_execute_env_receiver) = crossbeam_channel::unbounded();
 
     //let (pre_execute_env_sender, pre_execute_env_receiver) = crossbeam_channel::unbounded();
-    //let (post_execute_env_sender, post_execute_env_receiver) = crossbeam_channel::unbounded();
+    let (post_execute_env_sender, post_execute_env_receiver) = crossbeam_channel::unbounded();
     //
     let (post_schedule_env_sender, post_schedule_env_receiver) = crossbeam_channel::unbounded();
     let mut runnable_queue = TaskQueue::default();
@@ -282,6 +282,7 @@ fn output_slot(
                 &mut address_book,
                 &muxed_receiver,
                 &pre_execute_env_sender,
+                &post_execute_env_receiver,
                 Some(&post_schedule_env_sender),
             );
         })
@@ -327,9 +328,12 @@ fn output_slot(
                             );
                         }
 
+                        /*
                         muxed_sender
                             .send(solana_scheduler::Multiplexed::FromExecute(ee))
                             .unwrap();
+                        */
+                        post_execute_env_sender.send(ee).unwrap();
                     }
                 })
                 .unwrap();

From 95c98148c1e1f8d9e6f60168edc722a167148edc Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 15 Aug 2022 11:32:39 +0900
Subject: [PATCH 0661/3199] save

---
 ledger-tool/src/main.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ledger-tool/src/main.rs b/ledger-tool/src/main.rs
index 0a65f751acb351..6bbaf3e3d9546c 100644
--- a/ledger-tool/src/main.rs
+++ b/ledger-tool/src/main.rs
@@ -281,8 +281,8 @@ fn output_slot(
                 &mut contended_queue,
                 &mut address_book,
                 &muxed_receiver,
-                &pre_execute_env_sender,
                 &post_execute_env_receiver,
+                &pre_execute_env_sender,
                 Some(&post_schedule_env_sender),
             );
         })

From 34f82834c1b9c465be802e99e75d40b3c120e1eb Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 15 Aug 2022 11:33:09 +0900
Subject: [PATCH 0662/3199] save

---
 ledger-tool/src/main.rs | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/ledger-tool/src/main.rs b/ledger-tool/src/main.rs
index 6bbaf3e3d9546c..e8840b2b5fcd00 100644
--- a/ledger-tool/src/main.rs
+++ b/ledger-tool/src/main.rs
@@ -290,7 +290,8 @@ fn output_slot(
     let handles = (0..lane_count)
         .map(|thx| {
             let pre_execute_env_receiver = pre_execute_env_receiver.clone();
-            let muxed_sender = muxed_sender.clone();
+            let post_execute_env_sender = post_execute_env_sender.clone();
+            //let muxed_sender = muxed_sender.clone();
 
             let t2 = std::thread::Builder::new()
                 .name(format!("blockstore_processor_{}", thx))

From deb1fc7ef53f646ef9934d4bdb5cb07dd9ebbb1f Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 19 Aug 2022 17:20:42 +0900
Subject: [PATCH 0663/3199] save

---
 scheduler/src/lib.rs | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index edce2cc9561ed8..830a103bd65b38 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -667,6 +667,7 @@ impl ScheduleStage {
                     // maybe run lightweight prune logic on contended_queue here.
                 } else {
                     trace!("relock failed [{}/{}/{}]; remains in contended: {:?} contention: {}", unlockable_count, guaranteed_count, lock_count, &unique_weight, next_task.contention_count);
+                    contended_queue.clear();
                 }
 
                 continue;
@@ -682,8 +683,8 @@ impl ScheduleStage {
                 );
                 std::mem::swap(&mut next_task.tx.1, &mut populated_lock_attempts);
 
-                return None;
-                //continue;
+                //return None;
+                continue;
             }
 
             trace!("successful lock: (from_runnable: {}) after {} contentions", from_runnable, next_task.contention_count);

From 5d6aed1e52ed9802c73887668fa27a0a54ca53d7 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 19 Aug 2022 17:20:54 +0900
Subject: [PATCH 0664/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 830a103bd65b38..00c6a96d7c1747 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -667,7 +667,7 @@ impl ScheduleStage {
                     // maybe run lightweight prune logic on contended_queue here.
                 } else {
                     trace!("relock failed [{}/{}/{}]; remains in contended: {:?} contention: {}", unlockable_count, guaranteed_count, lock_count, &unique_weight, next_task.contention_count);
-                    contended_queue.clear();
+                    contended_queue.tasks.clear();
                 }
 
                 continue;

From 93bda18327b7dbfa06017c18b2ecffe4f00f41c5 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 19 Aug 2022 17:22:53 +0900
Subject: [PATCH 0665/3199] save

---
 scheduler/src/lib.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 00c6a96d7c1747..7aa3f954750c3b 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -876,7 +876,7 @@ impl ScheduleStage {
                         Multiplexed::FromPrevious(weighted_tx) => {
                             trace!("recv from previous");
 
-                            while from_exec.len() > 0 {
+                            while false && from_exec.len() > 0 {
                                 let mut processed_execution_environment = from_exec.recv().unwrap();
                                 trace!("recv from execute: {:?}", processed_execution_environment.unique_weight);
                                 executing_queue_count -= 1;
@@ -906,7 +906,7 @@ impl ScheduleStage {
                         if let Some(to_next_stage) = to_next_stage {
                             to_next_stage.send(processed_execution_environment).unwrap();
                         }
-                        if from_exec.len() > 0 {
+                        if false && from_exec.len() > 0 {
                             processed_execution_environment = from_exec.recv().unwrap();
                         } else {
                             break;

From 0fb8d21f6ccd806e1d4875f069802c5d19309d44 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 19 Aug 2022 17:24:46 +0900
Subject: [PATCH 0666/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 7aa3f954750c3b..d39c6f66030bda 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -667,7 +667,7 @@ impl ScheduleStage {
                     // maybe run lightweight prune logic on contended_queue here.
                 } else {
                     trace!("relock failed [{}/{}/{}]; remains in contended: {:?} contention: {}", unlockable_count, guaranteed_count, lock_count, &unique_weight, next_task.contention_count);
-                    contended_queue.tasks.clear();
+                    address_book.uncontended_task_ids.clear();
                 }
 
                 continue;

From 32e9bba81de86e76c41954c9b2f5313dd430b54d Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 19 Aug 2022 21:37:50 +0900
Subject: [PATCH 0667/3199] save

---
 scheduler/src/lib.rs | 12 ++++++++++++
 1 file changed, 12 insertions(+)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index d39c6f66030bda..ac68288f3f4e6f 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -934,6 +934,18 @@ impl ScheduleStage {
                     to_execute_substage.send(ee).unwrap();
                 } else {
                     trace!("incoming queue starved");
+                    while from.len() > 0 && from_exec.len() == 0 {
+                       let i = from.recv().unwrap();
+                        match i {
+                            Multiplexed::FromPrevious(weighted_tx) => {
+                                trace!("recv from previous (after starvation)");
+                                Self::register_runnable_task(weighted_tx, runnable_queue, &mut current_unique_key);
+                            }
+                            Multiplexed::FromPreviousBatched(vvv) => {
+                                unreachable!();
+                            }
+                        }
+                    }
                     break;
                 }
             }

From 25d4331eb5824d852b5434bcfd2811235fecf86a Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 19 Aug 2022 21:44:44 +0900
Subject: [PATCH 0668/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index ac68288f3f4e6f..5000bb6749ba23 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -923,7 +923,7 @@ impl ScheduleStage {
                     break;
                 }
 
-                let prefer_immediate = executing_queue_count < max_executing_queue_count / 2;
+                let prefer_immediate = address_book.gurantee_timers.len() > executing_queue_count;
                 let maybe_ee =
                     Self::schedule_next_execution(runnable_queue, contended_queue, address_book, prefer_immediate);
 

From a8bcecafaa3c7fc2203e5203a33da8eab22da821 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 19 Aug 2022 21:48:25 +0900
Subject: [PATCH 0669/3199] save

---
 scheduler/src/lib.rs | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 5000bb6749ba23..a63f1c9c04e8e5 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -670,7 +670,8 @@ impl ScheduleStage {
                     address_book.uncontended_task_ids.clear();
                 }
 
-                continue;
+                return None;
+                //continue;
             } else if guaranteed_count > 0 {
                 assert!(!from_runnable);
                 let lock_count = populated_lock_attempts.len();
@@ -683,7 +684,7 @@ impl ScheduleStage {
                 );
                 std::mem::swap(&mut next_task.tx.1, &mut populated_lock_attempts);
 
-                //return None;
+                return None;
                 continue;
             }
 

From 471d9ae38fad87ca72c5d54fb00fa6ca12bdff07 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 19 Aug 2022 21:50:58 +0900
Subject: [PATCH 0670/3199] save

---
 scheduler/src/lib.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index a63f1c9c04e8e5..1c43272aab66fb 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -586,10 +586,10 @@ impl ScheduleStage {
                 let weight_from_runnable = heaviest_runnable_entry.key();
                 let uw = weight_from_contended.key();
 
-                if weight_from_runnable > uw {
+                if true || weight_from_runnable > uw {
                     trace!("select: runnable > contended");
                     Some((Some(contended_queue), heaviest_runnable_entry))
-                } else if uw > weight_from_runnable {
+                } else if false && uw > weight_from_runnable {
                     trace!("select: contended > runnnable");
                     let uw = *uw;
                     weight_from_contended.remove();

From 8cf2f7dab3801bdae6b3fed34f317b2e2160fa30 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 19 Aug 2022 21:53:57 +0900
Subject: [PATCH 0671/3199] save

---
 scheduler/src/lib.rs | 13 +++++++++++++
 1 file changed, 13 insertions(+)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 1c43272aab66fb..872d13e75bbf08 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -890,6 +890,19 @@ impl ScheduleStage {
                             }
 
                             Self::register_runnable_task(weighted_tx, runnable_queue, &mut current_unique_key);
+
+                            while from.len() > 0 && from_exec.len() == 0 {
+                               let i = from.recv().unwrap();
+                                match i {
+                                    Multiplexed::FromPrevious(weighted_tx) => {
+                                        trace!("recv from previous (after starvation)");
+                                        Self::register_runnable_task(weighted_tx, runnable_queue, &mut current_unique_key);
+                                    }
+                                    Multiplexed::FromPreviousBatched(vvv) => {
+                                        unreachable!();
+                                    }
+                                }
+                            }
                         }
                         Multiplexed::FromPreviousBatched(vvv) => {
                             unreachable!();

From 4fda632a279c8d9d854b7ce6695ea1affee48e65 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 19 Aug 2022 21:56:03 +0900
Subject: [PATCH 0672/3199] save

---
 scheduler/src/lib.rs | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 872d13e75bbf08..e29c2df6881378 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -670,8 +670,11 @@ impl ScheduleStage {
                     address_book.uncontended_task_ids.clear();
                 }
 
-                return None;
-                //continue;
+                if from_runnable {
+                    continue;
+                } else {
+                    return None;
+                }
             } else if guaranteed_count > 0 {
                 assert!(!from_runnable);
                 let lock_count = populated_lock_attempts.len();

From feef07d7780ad4e5871115eab37fd9bb490392a5 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 19 Aug 2022 21:59:20 +0900
Subject: [PATCH 0673/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index e29c2df6881378..f4bed4b784c203 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -940,7 +940,7 @@ impl ScheduleStage {
                     break;
                 }
 
-                let prefer_immediate = address_book.gurantee_timers.len() > executing_queue_count;
+                let prefer_immediate = address_book.gurantee_timers.len()/4 > executing_queue_count;
                 let maybe_ee =
                     Self::schedule_next_execution(runnable_queue, contended_queue, address_book, prefer_immediate);
 

From 5716d8058a5773e1a65635c275a07cc95dc765c5 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 19 Aug 2022 22:15:52 +0900
Subject: [PATCH 0674/3199] save

---
 scheduler/src/lib.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index f4bed4b784c203..541467987513a8 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -933,9 +933,9 @@ impl ScheduleStage {
             }
 
             loop {
-                if !address_book.uncontended_task_ids.is_empty() {
+                /* if !address_book.uncontended_task_ids.is_empty() {
                     trace!("prefer emptying n_u_a");
-                } else if (executing_queue_count + address_book.gurantee_timers.len()) >= max_executing_queue_count {
+                } else */ if (executing_queue_count + address_book.gurantee_timers.len()) >= max_executing_queue_count {
                     trace!("skip scheduling; outgoing queue full");
                     break;
                 }

From 9985ee46046c16108d422b7f2d407362d94526f7 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 19 Aug 2022 22:19:31 +0900
Subject: [PATCH 0675/3199] save

---
 scheduler/src/lib.rs | 12 ++++++++++++
 1 file changed, 12 insertions(+)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 541467987513a8..a2c1dcf2f3140f 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -937,6 +937,18 @@ impl ScheduleStage {
                     trace!("prefer emptying n_u_a");
                 } else */ if (executing_queue_count + address_book.gurantee_timers.len()) >= max_executing_queue_count {
                     trace!("skip scheduling; outgoing queue full");
+                    while from.len() > 0 && from_exec.len() == 0 {
+                       let i = from.recv().unwrap();
+                        match i {
+                            Multiplexed::FromPrevious(weighted_tx) => {
+                                trace!("recv from previous (after starvation)");
+                                Self::register_runnable_task(weighted_tx, runnable_queue, &mut current_unique_key);
+                            }
+                            Multiplexed::FromPreviousBatched(vvv) => {
+                                unreachable!();
+                            }
+                        }
+                    }
                     break;
                 }
 

From b7279182e7fcc2b2805a64875d80cadb95d5b0e0 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 19 Aug 2022 22:21:22 +0900
Subject: [PATCH 0676/3199] save

---
 scheduler/src/lib.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index a2c1dcf2f3140f..ab616bcbe1f8e6 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -937,7 +937,7 @@ impl ScheduleStage {
                     trace!("prefer emptying n_u_a");
                 } else */ if (executing_queue_count + address_book.gurantee_timers.len()) >= max_executing_queue_count {
                     trace!("skip scheduling; outgoing queue full");
-                    while from.len() > 0 && from_exec.len() == 0 {
+                    while from.len() > 0 {
                        let i = from.recv().unwrap();
                         match i {
                             Multiplexed::FromPrevious(weighted_tx) => {
@@ -963,7 +963,7 @@ impl ScheduleStage {
                     to_execute_substage.send(ee).unwrap();
                 } else {
                     trace!("incoming queue starved");
-                    while from.len() > 0 && from_exec.len() == 0 {
+                    while from.len() > 0 {
                        let i = from.recv().unwrap();
                         match i {
                             Multiplexed::FromPrevious(weighted_tx) => {

From 45cf84f2e3e3d4116899d9220db6b2945b58ac9b Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 20 Aug 2022 13:54:28 +0900
Subject: [PATCH 0677/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index ab616bcbe1f8e6..fa92e8deb2bf6d 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -667,7 +667,7 @@ impl ScheduleStage {
                     // maybe run lightweight prune logic on contended_queue here.
                 } else {
                     trace!("relock failed [{}/{}/{}]; remains in contended: {:?} contention: {}", unlockable_count, guaranteed_count, lock_count, &unique_weight, next_task.contention_count);
-                    address_book.uncontended_task_ids.clear();
+                    //address_book.uncontended_task_ids.clear();
                 }
 
                 if from_runnable {

From 7e718d1c6e6ee605dc48b7a096badedb69974e3f Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 20 Aug 2022 14:04:41 +0900
Subject: [PATCH 0678/3199] save

---
 scheduler/Cargo.toml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/scheduler/Cargo.toml b/scheduler/Cargo.toml
index 37337b89997cc2..faf179afdf65d6 100644
--- a/scheduler/Cargo.toml
+++ b/scheduler/Cargo.toml
@@ -14,6 +14,7 @@ publish = true
 atomic_enum = "0.1.1"
 by_address = "1.0.4"
 crossbeam-channel = "0.5.5"
+crossbeam-skiplist = { path = "../../../crossbeam/crossbeam-skiplist" }
 dashmap = { version = "4.0.2" }
 rayon = "1.5.3"
 log = "0.4.17"

From 18902adec7fa420a74a5f97b8a2a904fd1ecc4b9 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 20 Aug 2022 14:05:37 +0900
Subject: [PATCH 0679/3199] save

---
 Cargo.lock           | 41 ++++++++++++++++++++++++++++++++++++-----
 scheduler/src/lib.rs |  3 ++-
 2 files changed, 38 insertions(+), 6 deletions(-)

diff --git a/Cargo.lock b/Cargo.lock
index 8f53bf829b134a..52d975f3e09388 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -1016,7 +1016,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "4c02a4d71819009c192cf4872265391563fd6a84c81ff2c0f2a7026ca4c1d85c"
 dependencies = [
  "cfg-if 1.0.0",
- "crossbeam-utils",
+ "crossbeam-utils 0.8.8",
 ]
 
 [[package]]
@@ -1026,8 +1026,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "6455c0ca19f0d2fbf751b908d5c55c1f5cbc65e03c4225427254b46890bdde1e"
 dependencies = [
  "cfg-if 1.0.0",
- "crossbeam-epoch",
- "crossbeam-utils",
+ "crossbeam-epoch 0.9.5",
+ "crossbeam-utils 0.8.8",
 ]
 
 [[package]]
@@ -1037,12 +1037,34 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "4ec02e091aa634e2c3ada4a392989e7c3116673ef0ac5b72232439094d73b7fd"
 dependencies = [
  "cfg-if 1.0.0",
- "crossbeam-utils",
+ "crossbeam-utils 0.8.8",
  "lazy_static",
  "memoffset",
  "scopeguard",
 ]
 
+[[package]]
+name = "crossbeam-epoch"
+version = "0.9.10"
+dependencies = [
+ "autocfg",
+ "cfg-if 1.0.0",
+ "crossbeam-utils 0.8.11",
+ "memoffset",
+ "once_cell",
+ "scopeguard",
+]
+
+[[package]]
+name = "crossbeam-skiplist"
+version = "0.0.0"
+dependencies = [
+ "cfg-if 1.0.0",
+ "crossbeam-epoch 0.9.10",
+ "crossbeam-utils 0.8.11",
+ "scopeguard",
+]
+
 [[package]]
 name = "crossbeam-utils"
 version = "0.8.8"
@@ -1053,6 +1075,14 @@ dependencies = [
  "lazy_static",
 ]
 
+[[package]]
+name = "crossbeam-utils"
+version = "0.8.11"
+dependencies = [
+ "cfg-if 1.0.0",
+ "once_cell",
+]
+
 [[package]]
 name = "crunchy"
 version = "0.2.2"
@@ -3800,7 +3830,7 @@ checksum = "9f51245e1e62e1f1629cbfec37b5793bbabcaeb90f30e94d2ba03564687353e4"
 dependencies = [
  "crossbeam-channel",
  "crossbeam-deque",
- "crossbeam-utils",
+ "crossbeam-utils 0.8.8",
  "num_cpus",
 ]
 
@@ -6075,6 +6105,7 @@ dependencies = [
  "atomic_enum",
  "by_address",
  "crossbeam-channel",
+ "crossbeam-skiplist",
  "dashmap",
  "log",
  "rand 0.7.3",
diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index fa92e8deb2bf6d..05c07b6af45eb4 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -115,7 +115,8 @@ pub enum RequestedUsage {
 
 #[derive(Debug, Default)]
 pub struct TaskIds {
-    task_ids: std::collections::BTreeSet<UniqueWeight>,
+    //task_ids: std::collections::BTreeSet<UniqueWeight>,
+    task_ids: crossbeam_skiplist::SkipSet<UniqueWeight>,
     cached_heaviest: Option<UniqueWeight>,
 }
 

From 2b5abb5a6bd69c1bd9dad644eb1ba2c7ec17625c Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 20 Aug 2022 14:06:47 +0900
Subject: [PATCH 0680/3199] save

---
 scheduler/src/lib.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 05c07b6af45eb4..b1ca0bcd522966 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -122,14 +122,14 @@ pub struct TaskIds {
 
 impl TaskIds {
     #[inline(never)]
-    fn insert(&mut self, u: UniqueWeight) -> bool {
+    fn insert(&mut self, u: UniqueWeight) {
         match self.cached_heaviest {
             Some(c) if u > c => { self.cached_heaviest = Some(u) },
             None => { self.cached_heaviest = Some(u); }
             _ => {},
         }
              
-        self.task_ids.insert(u)
+        self.task_ids.insert(u);
     }
 
     #[inline(never)]

From b3e91881b3a397356157d93f92bc8a72c4678530 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 20 Aug 2022 14:07:15 +0900
Subject: [PATCH 0681/3199] save

---
 scheduler/src/lib.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index b1ca0bcd522966..2217edfe27c32a 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -133,13 +133,13 @@ impl TaskIds {
     }
 
     #[inline(never)]
-    fn remove(&mut self, u: &UniqueWeight) -> bool {
+    fn remove(&mut self, u: &UniqueWeight) {
         let a = self.task_ids.remove(u);
         match self.cached_heaviest {
             Some(ref c) if u == c => { self.cached_heaviest = self.task_ids.last().copied() },
             _ => {},
         }
-        a
+        a;
     }
 
     #[inline(never)]

From f9f3ab8f81634cc2cec71672e11638869a4d640c Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 20 Aug 2022 14:09:11 +0900
Subject: [PATCH 0682/3199] save

---
 scheduler/src/lib.rs | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 2217edfe27c32a..22dbcc44c598ed 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -136,7 +136,8 @@ impl TaskIds {
     fn remove(&mut self, u: &UniqueWeight) {
         let a = self.task_ids.remove(u);
         match self.cached_heaviest {
-            Some(ref c) if u == c => { self.cached_heaviest = self.task_ids.last().copied() },
+            //Some(ref c) if u == c => { self.cached_heaviest = self.task_ids.last().copied() },
+            Some(ref c) if u == c => { self.cached_heaviest = self.task_ids.back().copied() },
             _ => {},
         }
         a;

From 8c13b8c4e103a744fc97ad22eb0d3126b68c3226 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 20 Aug 2022 14:10:18 +0900
Subject: [PATCH 0683/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 22dbcc44c598ed..5826ef6a8e7b1b 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -137,7 +137,7 @@ impl TaskIds {
         let a = self.task_ids.remove(u);
         match self.cached_heaviest {
             //Some(ref c) if u == c => { self.cached_heaviest = self.task_ids.last().copied() },
-            Some(ref c) if u == c => { self.cached_heaviest = self.task_ids.back().copied() },
+            Some(ref c) if u == c => { self.cached_heaviest = self.task_ids.back().value() },
             _ => {},
         }
         a;

From 87020c378ff6df5445b6762e12a6196e89fd5306 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 20 Aug 2022 14:11:26 +0900
Subject: [PATCH 0684/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 5826ef6a8e7b1b..ab725fa1ae603b 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -137,7 +137,7 @@ impl TaskIds {
         let a = self.task_ids.remove(u);
         match self.cached_heaviest {
             //Some(ref c) if u == c => { self.cached_heaviest = self.task_ids.last().copied() },
-            Some(ref c) if u == c => { self.cached_heaviest = self.task_ids.back().value() },
+            Some(ref c) if u == c => { self.cached_heaviest = self.task_ids.back().map(|e| e.value()) },
             _ => {},
         }
         a;

From 014453f25bbeec13f3826d159fc95e1eaf6db57c Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 20 Aug 2022 14:11:37 +0900
Subject: [PATCH 0685/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index ab725fa1ae603b..82de9f84b15777 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -137,7 +137,7 @@ impl TaskIds {
         let a = self.task_ids.remove(u);
         match self.cached_heaviest {
             //Some(ref c) if u == c => { self.cached_heaviest = self.task_ids.last().copied() },
-            Some(ref c) if u == c => { self.cached_heaviest = self.task_ids.back().map(|e| e.value()) },
+            Some(ref c) if u == c => { self.cached_heaviest = self.task_ids.back().map(|e| e.value()).copied() },
             _ => {},
         }
         a;

From e52afa4dbd83f3f0a42e30c5a73c3b641197045d Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 20 Aug 2022 14:12:11 +0900
Subject: [PATCH 0686/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 82de9f84b15777..d57e79263bf892 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -137,7 +137,7 @@ impl TaskIds {
         let a = self.task_ids.remove(u);
         match self.cached_heaviest {
             //Some(ref c) if u == c => { self.cached_heaviest = self.task_ids.last().copied() },
-            Some(ref c) if u == c => { self.cached_heaviest = self.task_ids.back().map(|e| e.value()).copied() },
+            Some(ref c) if u == c => { self.cached_heaviest = self.task_ids.back().map(|e| e.value().copied()) },
             _ => {},
         }
         a;

From 7a338e0def1bd2618c1f4f8cb804b29243e3009c Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 20 Aug 2022 14:12:33 +0900
Subject: [PATCH 0687/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index d57e79263bf892..c08552ccc797d1 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -137,7 +137,7 @@ impl TaskIds {
         let a = self.task_ids.remove(u);
         match self.cached_heaviest {
             //Some(ref c) if u == c => { self.cached_heaviest = self.task_ids.last().copied() },
-            Some(ref c) if u == c => { self.cached_heaviest = self.task_ids.back().map(|e| e.value().copied()) },
+            Some(ref c) if u == c => { self.cached_heaviest = self.task_ids.back().map(|e| *(e.value())) },
             _ => {},
         }
         a;

From 2ba70e749ab1e00312bf25744eab1c84cbacacd2 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 20 Aug 2022 14:21:03 +0900
Subject: [PATCH 0688/3199] save

---
 scheduler/src/lib.rs | 11 ++++++++---
 1 file changed, 8 insertions(+), 3 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index c08552ccc797d1..91f1ab2c7b5fbc 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -117,17 +117,19 @@ pub enum RequestedUsage {
 pub struct TaskIds {
     //task_ids: std::collections::BTreeSet<UniqueWeight>,
     task_ids: crossbeam_skiplist::SkipSet<UniqueWeight>,
-    cached_heaviest: Option<UniqueWeight>,
+    //cached_heaviest: Option<UniqueWeight>,
 }
 
 impl TaskIds {
     #[inline(never)]
     fn insert(&mut self, u: UniqueWeight) {
+        /*
         match self.cached_heaviest {
             Some(c) if u > c => { self.cached_heaviest = Some(u) },
             None => { self.cached_heaviest = Some(u); }
             _ => {},
         }
+        */
              
         self.task_ids.insert(u);
     }
@@ -135,11 +137,13 @@ impl TaskIds {
     #[inline(never)]
     fn remove(&mut self, u: &UniqueWeight) {
         let a = self.task_ids.remove(u);
+        /*
         match self.cached_heaviest {
             //Some(ref c) if u == c => { self.cached_heaviest = self.task_ids.last().copied() },
             Some(ref c) if u == c => { self.cached_heaviest = self.task_ids.back().map(|e| *(e.value())) },
             _ => {},
         }
+        */
         a;
     }
 
@@ -149,9 +153,10 @@ impl TaskIds {
     }
 
     #[inline(never)]
-    fn last(&self) -> Option<&UniqueWeight> {
+    fn last(&self) -> Option<UniqueWeight> {
         //self.task_ids.last()
-        self.cached_heaviest.as_ref()
+        self.task_ids.back().map(|e| *(e.value()))
+        //self.cached_heaviest.as_ref()
     }
 }
 

From 3ed8369bc85616ff3da630963321d8a83582f66f Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 20 Aug 2022 14:21:25 +0900
Subject: [PATCH 0689/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 91f1ab2c7b5fbc..b26b0d3912801c 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -155,7 +155,7 @@ impl TaskIds {
     #[inline(never)]
     fn last(&self) -> Option<UniqueWeight> {
         //self.task_ids.last()
-        self.task_ids.back().map(|e| *(e.value()))
+        self.task_ids.back().map(|e| e.value())
         //self.cached_heaviest.as_ref()
     }
 }

From 58d66bf5dd2a0e75f6ad0b3f5723b836f915ee59 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 20 Aug 2022 14:21:35 +0900
Subject: [PATCH 0690/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index b26b0d3912801c..8769b025415653 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -153,7 +153,7 @@ impl TaskIds {
     }
 
     #[inline(never)]
-    fn last(&self) -> Option<UniqueWeight> {
+    fn last(&self) -> Option<&UniqueWeight> {
         //self.task_ids.last()
         self.task_ids.back().map(|e| e.value())
         //self.cached_heaviest.as_ref()

From 350540fea7f67c090afe72b48bf913add418a38d Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 20 Aug 2022 14:21:43 +0900
Subject: [PATCH 0691/3199] save

---
 scheduler/src/lib.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 8769b025415653..91f1ab2c7b5fbc 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -153,9 +153,9 @@ impl TaskIds {
     }
 
     #[inline(never)]
-    fn last(&self) -> Option<&UniqueWeight> {
+    fn last(&self) -> Option<UniqueWeight> {
         //self.task_ids.last()
-        self.task_ids.back().map(|e| e.value())
+        self.task_ids.back().map(|e| *(e.value()))
         //self.cached_heaviest.as_ref()
     }
 }

From c43aed5c4c9ef9acb9d60b9250fdea690f5a123f Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 20 Aug 2022 14:21:51 +0900
Subject: [PATCH 0692/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 91f1ab2c7b5fbc..e1638672bd9836 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -781,7 +781,7 @@ impl ScheduleStage {
             let page = l.target.page();
             if newly_uncontended_while_queued && page.next_usage == Usage::Unused {
                 if let Some(uw) = page.contended_unique_weights.last() {
-                    address_book.uncontended_task_ids.insert(*uw, ());
+                    address_book.uncontended_task_ids.insert(uw, ());
                 }
             }
             if page.current_usage == Usage::Unused && page.next_usage != Usage::Unused {

From f61673994f21bf7cebe53e3da1ba6ae6a3cd7961 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 20 Aug 2022 14:25:08 +0900
Subject: [PATCH 0693/3199] save

---
 scheduler/src/lib.rs | 13 ++++---------
 1 file changed, 4 insertions(+), 9 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index e1638672bd9836..c08552ccc797d1 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -117,19 +117,17 @@ pub enum RequestedUsage {
 pub struct TaskIds {
     //task_ids: std::collections::BTreeSet<UniqueWeight>,
     task_ids: crossbeam_skiplist::SkipSet<UniqueWeight>,
-    //cached_heaviest: Option<UniqueWeight>,
+    cached_heaviest: Option<UniqueWeight>,
 }
 
 impl TaskIds {
     #[inline(never)]
     fn insert(&mut self, u: UniqueWeight) {
-        /*
         match self.cached_heaviest {
             Some(c) if u > c => { self.cached_heaviest = Some(u) },
             None => { self.cached_heaviest = Some(u); }
             _ => {},
         }
-        */
              
         self.task_ids.insert(u);
     }
@@ -137,13 +135,11 @@ impl TaskIds {
     #[inline(never)]
     fn remove(&mut self, u: &UniqueWeight) {
         let a = self.task_ids.remove(u);
-        /*
         match self.cached_heaviest {
             //Some(ref c) if u == c => { self.cached_heaviest = self.task_ids.last().copied() },
             Some(ref c) if u == c => { self.cached_heaviest = self.task_ids.back().map(|e| *(e.value())) },
             _ => {},
         }
-        */
         a;
     }
 
@@ -153,10 +149,9 @@ impl TaskIds {
     }
 
     #[inline(never)]
-    fn last(&self) -> Option<UniqueWeight> {
+    fn last(&self) -> Option<&UniqueWeight> {
         //self.task_ids.last()
-        self.task_ids.back().map(|e| *(e.value()))
-        //self.cached_heaviest.as_ref()
+        self.cached_heaviest.as_ref()
     }
 }
 
@@ -781,7 +776,7 @@ impl ScheduleStage {
             let page = l.target.page();
             if newly_uncontended_while_queued && page.next_usage == Usage::Unused {
                 if let Some(uw) = page.contended_unique_weights.last() {
-                    address_book.uncontended_task_ids.insert(uw, ());
+                    address_book.uncontended_task_ids.insert(*uw, ());
                 }
             }
             if page.current_usage == Usage::Unused && page.next_usage != Usage::Unused {

From f0fe6d35811f8170c8afec6d64c0e97cf2c48b5d Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 20 Aug 2022 14:27:31 +0900
Subject: [PATCH 0694/3199] save

---
 scheduler/src/lib.rs | 13 +++++++++----
 1 file changed, 9 insertions(+), 4 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index c08552ccc797d1..e1638672bd9836 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -117,17 +117,19 @@ pub enum RequestedUsage {
 pub struct TaskIds {
     //task_ids: std::collections::BTreeSet<UniqueWeight>,
     task_ids: crossbeam_skiplist::SkipSet<UniqueWeight>,
-    cached_heaviest: Option<UniqueWeight>,
+    //cached_heaviest: Option<UniqueWeight>,
 }
 
 impl TaskIds {
     #[inline(never)]
     fn insert(&mut self, u: UniqueWeight) {
+        /*
         match self.cached_heaviest {
             Some(c) if u > c => { self.cached_heaviest = Some(u) },
             None => { self.cached_heaviest = Some(u); }
             _ => {},
         }
+        */
              
         self.task_ids.insert(u);
     }
@@ -135,11 +137,13 @@ impl TaskIds {
     #[inline(never)]
     fn remove(&mut self, u: &UniqueWeight) {
         let a = self.task_ids.remove(u);
+        /*
         match self.cached_heaviest {
             //Some(ref c) if u == c => { self.cached_heaviest = self.task_ids.last().copied() },
             Some(ref c) if u == c => { self.cached_heaviest = self.task_ids.back().map(|e| *(e.value())) },
             _ => {},
         }
+        */
         a;
     }
 
@@ -149,9 +153,10 @@ impl TaskIds {
     }
 
     #[inline(never)]
-    fn last(&self) -> Option<&UniqueWeight> {
+    fn last(&self) -> Option<UniqueWeight> {
         //self.task_ids.last()
-        self.cached_heaviest.as_ref()
+        self.task_ids.back().map(|e| *(e.value()))
+        //self.cached_heaviest.as_ref()
     }
 }
 
@@ -776,7 +781,7 @@ impl ScheduleStage {
             let page = l.target.page();
             if newly_uncontended_while_queued && page.next_usage == Usage::Unused {
                 if let Some(uw) = page.contended_unique_weights.last() {
-                    address_book.uncontended_task_ids.insert(*uw, ());
+                    address_book.uncontended_task_ids.insert(uw, ());
                 }
             }
             if page.current_usage == Usage::Unused && page.next_usage != Usage::Unused {

From ace8f918d0b1e5b5650825f39e0bb34c2a297d77 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 20 Aug 2022 14:41:21 +0900
Subject: [PATCH 0695/3199] save

---
 scheduler/src/lib.rs | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index e1638672bd9836..1999a0b9f9f6ab 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -788,12 +788,12 @@ impl ScheduleStage {
                 page.switch_to_next_usage();
                 for task_id in std::mem::take(&mut page.guaranteed_task_ids).keys() {
                     match address_book.gurantee_timers.entry(*task_id) {
-                        std::collections::hash_map::Entry::Occupied(mut entry) => {
-                            let count = entry.get_mut();
+                        std::collections::hash_map::Entry::Occupied(mut timer_entry) => {
+                            let count = timer_entry.get_mut();
                             *count = count.checked_sub(1).unwrap();
                             if *count == 0 {
                                 trace!("guaranteed lock decrease: {} => {} (!)", *count + 1, *count);
-                                entry.remove();
+                                timer_entry.remove();
                                 address_book.runnable_guaranteed_task_ids.insert(*task_id, ());
                             } else {
                                 trace!("guaranteed lock decrease: {} => {}", *count + 1, *count);

From a1a1216a742036c56c1fa62a8c8ab3a494275ad0 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 20 Aug 2022 14:54:49 +0900
Subject: [PATCH 0696/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 1999a0b9f9f6ab..a7fed8d0b5ece9 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -213,7 +213,7 @@ impl AddressBook {
     ) {
         let LockAttempt {target, requested_usage, status, remembered} = attempt;
 
-                let mut page = unsafe { MyRcInner::get_mut_unchecked(&mut target.0) };
+                let mut page = target.page();
 
                 match page.current_usage {
                     Usage::Unused => {

From 7e0a10713e04c5c4ea45f2b79d89025a76f9f148 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 20 Aug 2022 14:55:30 +0900
Subject: [PATCH 0697/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index a7fed8d0b5ece9..e883793c18ee6f 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -51,7 +51,7 @@ impl ExecutionEnvironment {
 }
 
 impl PageRc {
-    fn page(&mut self) -> &mut Page {
+    fn page_mut(&mut self) -> &mut Page {
         unsafe { MyRcInner::get_mut_unchecked(&mut self.0) }
     }
 }

From c841d6c664db51df96bab080385e04b2a436428f Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 20 Aug 2022 14:55:49 +0900
Subject: [PATCH 0698/3199] save

---
 scheduler/src/lib.rs | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index e883793c18ee6f..921c7b636a00e6 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -738,7 +738,7 @@ impl ScheduleStage {
             AddressBook::forget_address_contention(&unique_weight, &mut l);
             match l.status {
                 LockStatus::Guaranteed => {
-                    l.target.page().guaranteed_task_ids.insert(*unique_weight, ());
+                    l.target.page_mut().guaranteed_task_ids.insert(*unique_weight, ());
                 }
                 LockStatus::Succeded => {
                     // do nothing
@@ -761,7 +761,7 @@ impl ScheduleStage {
     ) {
         for l in lock_attempts {
             if from_runnable {
-                AddressBook::remember_address_contention(&mut l.target.page(), unique_weight);
+                AddressBook::remember_address_contention(&mut l.target.page_mut(), unique_weight);
                 l.remembered = true;
             }
             address_book.reset_lock(l, false);
@@ -778,7 +778,7 @@ impl ScheduleStage {
         for mut l in lock_attempts.into_iter() {
             let newly_uncontended_while_queued = address_book.reset_lock(&mut l, true);
 
-            let page = l.target.page();
+            let page = l.target.page_mut();
             if newly_uncontended_while_queued && page.next_usage == Usage::Unused {
                 if let Some(uw) = page.contended_unique_weights.last() {
                     address_book.uncontended_task_ids.insert(uw, ());

From e19627e72e0513a723ba77ecfd7513fee25bd2c0 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 20 Aug 2022 14:56:08 +0900
Subject: [PATCH 0699/3199] save

---
 scheduler/src/lib.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 921c7b636a00e6..cde7315795234a 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -315,7 +315,7 @@ impl AddressBook {
         let mut newly_uncontended = false;
         let mut still_queued = false;
 
-        let mut page = attempt.target.page();
+        let mut page = attempt.target.page_mut();
 
         match &mut page.current_usage {
             Usage::Readonly(ref mut count) => match &attempt.requested_usage {
@@ -349,7 +349,7 @@ impl AddressBook {
 
     #[inline(never)]
     fn cancel(&mut self, attempt: &mut LockAttempt) {
-        let mut page = attempt.target.page();
+        let mut page = attempt.target.page_mut();
 
         match page.next_usage {
             Usage::Unused => {

From f8fe76fe87df1f42510e456c46f729537a0da241 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 20 Aug 2022 14:56:17 +0900
Subject: [PATCH 0700/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index cde7315795234a..436a6ec00c7688 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -285,7 +285,7 @@ impl AddressBook {
     #[inline(never)]
     fn forget_address_contention(unique_weight: &UniqueWeight, a: &mut LockAttempt) {
         if a.remembered {
-            a.target.page().contended_unique_weights.remove(unique_weight);
+            a.target.page_mut().contended_unique_weights.remove(unique_weight);
         }
     }
 

From d634244468bcc2314fdaf0af8785419cf9ec3a3d Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 20 Aug 2022 14:56:25 +0900
Subject: [PATCH 0701/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 436a6ec00c7688..f00ed57349c065 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -213,7 +213,7 @@ impl AddressBook {
     ) {
         let LockAttempt {target, requested_usage, status, remembered} = attempt;
 
-                let mut page = target.page();
+                let mut page = target.page_mut();
 
                 match page.current_usage {
                     Usage::Unused => {

From b4a7c4dbf2d0c51c47351952c7c127bb2b3ee178 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 20 Aug 2022 14:58:40 +0900
Subject: [PATCH 0702/3199] save

---
 scheduler/src/lib.rs | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index f00ed57349c065..d09cf67e029d29 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -54,6 +54,10 @@ impl PageRc {
     fn page_mut(&mut self) -> &mut Page {
         unsafe { MyRcInner::get_mut_unchecked(&mut self.0) }
     }
+
+    fn page_ref(&self) -> &Page {
+        MyRcInner::get_ref(&mut self.0)
+    }
 }
 
 #[derive(Clone, Debug)]

From 80fe07ada04332057ffdd2356373a67f5645aa4f Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 20 Aug 2022 14:58:49 +0900
Subject: [PATCH 0703/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index d09cf67e029d29..4f6e5143f10ee3 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -56,7 +56,7 @@ impl PageRc {
     }
 
     fn page_ref(&self) -> &Page {
-        MyRcInner::get_ref(&mut self.0)
+        MyRcInner::ref(&mut self.0)
     }
 }
 

From 0a8842253306fbffb32525719a6944b8e9431d26 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 20 Aug 2022 14:58:57 +0900
Subject: [PATCH 0704/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 4f6e5143f10ee3..30c60d6fc46861 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -56,7 +56,7 @@ impl PageRc {
     }
 
     fn page_ref(&self) -> &Page {
-        MyRcInner::ref(&mut self.0)
+        MyRcInner::get(&mut self.0)
     }
 }
 

From afd08e3dad6713d45d00442006de3b1eafd50832 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 20 Aug 2022 14:59:48 +0900
Subject: [PATCH 0705/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 30c60d6fc46861..b8cb6594caa6e6 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -56,7 +56,7 @@ impl PageRc {
     }
 
     fn page_ref(&self) -> &Page {
-        MyRcInner::get(&mut self.0)
+        *(self.0)
     }
 }
 

From c2d7311e86cc8a1e5db13ce0064b4723292be73f Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 20 Aug 2022 15:00:01 +0900
Subject: [PATCH 0706/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index b8cb6594caa6e6..d7a64405b8051b 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -56,7 +56,7 @@ impl PageRc {
     }
 
     fn page_ref(&self) -> &Page {
-        *(self.0)
+        &*(self.0)
     }
 }
 

From a2c452b1d3fd3b9e452f137f13e790d24194bc36 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 20 Aug 2022 15:01:23 +0900
Subject: [PATCH 0707/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index d7a64405b8051b..afd0f7ad14cdf3 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -56,7 +56,7 @@ impl PageRc {
     }
 
     fn page_ref(&self) -> &Page {
-        &*(self.0)
+        self.0.borrow()
     }
 }
 

From 213690fdfa5728d5cf47a7d9ddc23ea3376366e2 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 20 Aug 2022 15:01:43 +0900
Subject: [PATCH 0708/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index afd0f7ad14cdf3..54bd55e2557636 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -56,7 +56,7 @@ impl PageRc {
     }
 
     fn page_ref(&self) -> &Page {
-        self.0.borrow()
+        Arc::borrow(self.0)
     }
 }
 

From 75a0daf3980f7aa94366ef855205bb39d5e8ef71 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 20 Aug 2022 15:01:54 +0900
Subject: [PATCH 0709/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 54bd55e2557636..8d04ea6a21ef10 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -56,7 +56,7 @@ impl PageRc {
     }
 
     fn page_ref(&self) -> &Page {
-        Arc::borrow(self.0)
+        MyRcInner::borrow(self.0)
     }
 }
 

From 8738d55b238644c4a24004c1a5b08adff8cfda1a Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 20 Aug 2022 15:02:10 +0900
Subject: [PATCH 0710/3199] save

---
 scheduler/src/lib.rs | 1 +
 1 file changed, 1 insertion(+)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 8d04ea6a21ef10..ee1fc01e634499 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -56,6 +56,7 @@ impl PageRc {
     }
 
     fn page_ref(&self) -> &Page {
+        use std::borrow::Borrow;
         MyRcInner::borrow(self.0)
     }
 }

From 43a7331f1e8e4d3acf3edd331baaf3895afa45d3 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 20 Aug 2022 15:02:18 +0900
Subject: [PATCH 0711/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index ee1fc01e634499..49793dff76d63c 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -57,7 +57,7 @@ impl PageRc {
 
     fn page_ref(&self) -> &Page {
         use std::borrow::Borrow;
-        MyRcInner::borrow(self.0)
+        MyRcInner::borrow(&self.0)
     }
 }
 

From 1c0e887aca2547c3b27162c60610d18d76c9d84f Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 20 Aug 2022 15:03:01 +0900
Subject: [PATCH 0712/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 49793dff76d63c..684a021c82b2b8 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -57,7 +57,7 @@ impl PageRc {
 
     fn page_ref(&self) -> &Page {
         use std::borrow::Borrow;
-        MyRcInner::borrow(&self.0)
+        <MyRcInner as std::borrow::Borrow>::borrow(&self.0)
     }
 }
 

From 2e267f82cea4525f2631f31de27b11a504e1e344 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 20 Aug 2022 15:03:19 +0900
Subject: [PATCH 0713/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 684a021c82b2b8..5a1d2b3cdb47a8 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -57,7 +57,7 @@ impl PageRc {
 
     fn page_ref(&self) -> &Page {
         use std::borrow::Borrow;
-        <MyRcInner as std::borrow::Borrow>::borrow(&self.0)
+        <MyRcInner as std::borrow::Borrow<_>>::borrow(&self.0)
     }
 }
 

From 58ced5021651db3e8106657cbd3d08b9b9daabe6 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 20 Aug 2022 15:03:43 +0900
Subject: [PATCH 0714/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 5a1d2b3cdb47a8..dda0f551170f0a 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -57,7 +57,7 @@ impl PageRc {
 
     fn page_ref(&self) -> &Page {
         use std::borrow::Borrow;
-        <MyRcInner as std::borrow::Borrow<_>>::borrow(&self.0)
+        <MyRcInner<_> as std::borrow::Borrow<_>>::borrow(&self.0)
     }
 }
 

From b13215e3e79dd2aefdc13ad238ae3676dc760344 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 20 Aug 2022 15:03:51 +0900
Subject: [PATCH 0715/3199] save

---
 scheduler/src/lib.rs | 1 -
 1 file changed, 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index dda0f551170f0a..b3bb31dd70d21f 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -56,7 +56,6 @@ impl PageRc {
     }
 
     fn page_ref(&self) -> &Page {
-        use std::borrow::Borrow;
         <MyRcInner<_> as std::borrow::Borrow<_>>::borrow(&self.0)
     }
 }

From 55d71c531a982340bf98749442cb4dfaeb0d2bd0 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 20 Aug 2022 15:04:17 +0900
Subject: [PATCH 0716/3199] save

---
 scheduler/src/lib.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index b3bb31dd70d21f..d5faddaa113bd5 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -22,10 +22,10 @@ type MyRcInner<T> = std::rc::Rc<T>;
 unsafe impl Send for PageRc {}
 */
 
-type MyRcInner<T> = std::sync::Arc<T>;
+type MyRcInner<Page> = std::sync::Arc<Page>;
 
 #[derive(Debug, Clone)]
-pub struct PageRc(MyRcInner<Page>);
+pub struct PageRc(MyRcInner);
 
 
 #[derive(Debug)]

From e6d3339369324fa2aac00d7bc141cbaba25b3b08 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 20 Aug 2022 15:04:31 +0900
Subject: [PATCH 0717/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index d5faddaa113bd5..cc14371dc0c8f4 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -22,7 +22,7 @@ type MyRcInner<T> = std::rc::Rc<T>;
 unsafe impl Send for PageRc {}
 */
 
-type MyRcInner<Page> = std::sync::Arc<Page>;
+type MyRcInner = std::sync::Arc<Page>;
 
 #[derive(Debug, Clone)]
 pub struct PageRc(MyRcInner);

From 93a01056be7b404ed960e376e0d7cb32263f87a4 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 20 Aug 2022 15:04:38 +0900
Subject: [PATCH 0718/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index cc14371dc0c8f4..59679b082cd4b4 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -56,7 +56,7 @@ impl PageRc {
     }
 
     fn page_ref(&self) -> &Page {
-        <MyRcInner<_> as std::borrow::Borrow<_>>::borrow(&self.0)
+        <MyRcInner as std::borrow::Borrow<_>>::borrow(&self.0)
     }
 }
 

From d156d659e1d42c4059374d779ce7d69912d84967 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 20 Aug 2022 15:05:08 +0900
Subject: [PATCH 0719/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 59679b082cd4b4..1289e8445f031f 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -56,7 +56,7 @@ impl PageRc {
     }
 
     fn page_ref(&self) -> &Page {
-        <MyRcInner as std::borrow::Borrow<_>>::borrow(&self.0)
+        <MyRcInner as std::borrow::Borrow>::borrow(&self.0)
     }
 }
 

From 87cbf5822a267289ab1c63a814be6eb71f720582 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 20 Aug 2022 15:05:21 +0900
Subject: [PATCH 0720/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 1289e8445f031f..59679b082cd4b4 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -56,7 +56,7 @@ impl PageRc {
     }
 
     fn page_ref(&self) -> &Page {
-        <MyRcInner as std::borrow::Borrow>::borrow(&self.0)
+        <MyRcInner as std::borrow::Borrow<_>>::borrow(&self.0)
     }
 }
 

From 398be4282858fbf2b38d86d874666664ddaf06cc Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 20 Aug 2022 15:08:12 +0900
Subject: [PATCH 0721/3199] save

---
 transaction-scheduler-bench/src/main.rs | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/transaction-scheduler-bench/src/main.rs b/transaction-scheduler-bench/src/main.rs
index 52e82a30e9e973..fc046aa16de766 100644
--- a/transaction-scheduler-bench/src/main.rs
+++ b/transaction-scheduler-bench/src/main.rs
@@ -345,6 +345,9 @@ fn handle_transaction_batch(
         .fetch_add(priority_collected, Ordering::Relaxed);
 
     let uq = transaction_batch.unique_weight;
+    for lock_attempt in transaction_batch.lock_attempts {
+        lock_attempt.page_ref();
+    }
     completed_transaction_sender.0
         .send(transaction_batch)
         .unwrap();

From 7f62f2c438f01d11d7f627d12a670e00a0704c2d Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 20 Aug 2022 15:08:26 +0900
Subject: [PATCH 0722/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 59679b082cd4b4..b7ee12e1bf5b81 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -30,7 +30,7 @@ pub struct PageRc(MyRcInner);
 
 #[derive(Debug)]
 pub struct ExecutionEnvironment {
-    lock_attempts: Vec<LockAttempt>,
+    pub lock_attempts: Vec<LockAttempt>,
     //accounts: Vec<i8>,
     pub cu: usize,
     pub unique_weight: UniqueWeight,

From 599fa460905512b05b7bf7ac489a2ac336317f07 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 20 Aug 2022 15:08:48 +0900
Subject: [PATCH 0723/3199] save

---
 transaction-scheduler-bench/src/main.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/transaction-scheduler-bench/src/main.rs b/transaction-scheduler-bench/src/main.rs
index fc046aa16de766..e4523dfa5c6781 100644
--- a/transaction-scheduler-bench/src/main.rs
+++ b/transaction-scheduler-bench/src/main.rs
@@ -346,7 +346,7 @@ fn handle_transaction_batch(
 
     let uq = transaction_batch.unique_weight;
     for lock_attempt in transaction_batch.lock_attempts {
-        lock_attempt.page_ref();
+        lock_attempt.target.page_ref();
     }
     completed_transaction_sender.0
         .send(transaction_batch)

From 68d36e7611dc1dd9b88fd55d433e38426df21aaa Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 20 Aug 2022 15:09:01 +0900
Subject: [PATCH 0724/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index b7ee12e1bf5b81..f8c51ef59ca87b 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -55,7 +55,7 @@ impl PageRc {
         unsafe { MyRcInner::get_mut_unchecked(&mut self.0) }
     }
 
-    fn page_ref(&self) -> &Page {
+    pub fn page_ref(&self) -> &Page {
         <MyRcInner as std::borrow::Borrow<_>>::borrow(&self.0)
     }
 }

From b37808e445e29d78c82bbcafbb557ba623757caa Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 20 Aug 2022 15:09:16 +0900
Subject: [PATCH 0725/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index f8c51ef59ca87b..957b4531e47130 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -165,7 +165,7 @@ impl TaskIds {
 }
 
 #[derive(Debug)]
-struct Page {
+pub struct Page {
     current_usage: Usage,
     next_usage: Usage,
     contended_unique_weights: TaskIds,

From dfb0b563df92bf88de91d9cd285c635e88276418 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 20 Aug 2022 15:09:33 +0900
Subject: [PATCH 0726/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 957b4531e47130..0be334a28dffbe 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -69,7 +69,7 @@ enum LockStatus {
 
 #[derive(Clone, Debug)]
 pub struct LockAttempt {
-    target: PageRc,
+    pub target: PageRc,
     status: LockStatus,
     requested_usage: RequestedUsage,
     remembered: bool,

From 23ebbab6b53478b6516fa0232acb37cf8a78264e Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 20 Aug 2022 15:09:48 +0900
Subject: [PATCH 0727/3199] save

---
 transaction-scheduler-bench/src/main.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/transaction-scheduler-bench/src/main.rs b/transaction-scheduler-bench/src/main.rs
index e4523dfa5c6781..3b6845a7bd6110 100644
--- a/transaction-scheduler-bench/src/main.rs
+++ b/transaction-scheduler-bench/src/main.rs
@@ -345,7 +345,7 @@ fn handle_transaction_batch(
         .fetch_add(priority_collected, Ordering::Relaxed);
 
     let uq = transaction_batch.unique_weight;
-    for lock_attempt in transaction_batch.lock_attempts {
+    for lock_attempt in transaction_batch.lock_attempts.iter() {
         lock_attempt.target.page_ref();
     }
     completed_transaction_sender.0

From aa9af6bf40fb3116f26ee7d3dafd69b36a471a8f Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 20 Aug 2022 15:10:16 +0900
Subject: [PATCH 0728/3199] save

---
 transaction-scheduler-bench/src/main.rs | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/transaction-scheduler-bench/src/main.rs b/transaction-scheduler-bench/src/main.rs
index 3b6845a7bd6110..b3e3b22d8844a9 100644
--- a/transaction-scheduler-bench/src/main.rs
+++ b/transaction-scheduler-bench/src/main.rs
@@ -346,7 +346,8 @@ fn handle_transaction_batch(
 
     let uq = transaction_batch.unique_weight;
     for lock_attempt in transaction_batch.lock_attempts.iter() {
-        lock_attempt.target.page_ref();
+        let page = lock_attempt.target.page_ref();
+        page.uncontended_task_ids
     }
     completed_transaction_sender.0
         .send(transaction_batch)

From af99eb8009f223cbcb4f11169beb8c94ff147b9e Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 20 Aug 2022 15:10:49 +0900
Subject: [PATCH 0729/3199] save

---
 transaction-scheduler-bench/src/main.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/transaction-scheduler-bench/src/main.rs b/transaction-scheduler-bench/src/main.rs
index b3e3b22d8844a9..e9d30d1267e8d2 100644
--- a/transaction-scheduler-bench/src/main.rs
+++ b/transaction-scheduler-bench/src/main.rs
@@ -347,7 +347,7 @@ fn handle_transaction_batch(
     let uq = transaction_batch.unique_weight;
     for lock_attempt in transaction_batch.lock_attempts.iter() {
         let page = lock_attempt.target.page_ref();
-        page.uncontended_task_ids
+        page.contended_unique_weights
     }
     completed_transaction_sender.0
         .send(transaction_batch)

From 23b713d745511d29f709c7d2ee6a5478e940bc1f Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 20 Aug 2022 15:10:57 +0900
Subject: [PATCH 0730/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 0be334a28dffbe..b63599d3a609f9 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -168,7 +168,7 @@ impl TaskIds {
 pub struct Page {
     current_usage: Usage,
     next_usage: Usage,
-    contended_unique_weights: TaskIds,
+    pub contended_unique_weights: TaskIds,
     guaranteed_task_ids: WeightedTaskIds,
     //loaded account from Accounts db
     //comulative_cu for qos; i.e. track serialized cumulative keyed by addresses and bail out block

From 61811768b0c8aeb29b85dbc7a34cf96490554864 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 20 Aug 2022 15:11:35 +0900
Subject: [PATCH 0731/3199] save

---
 transaction-scheduler-bench/src/main.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/transaction-scheduler-bench/src/main.rs b/transaction-scheduler-bench/src/main.rs
index e9d30d1267e8d2..4cdb1ad6e006f3 100644
--- a/transaction-scheduler-bench/src/main.rs
+++ b/transaction-scheduler-bench/src/main.rs
@@ -347,7 +347,7 @@ fn handle_transaction_batch(
     let uq = transaction_batch.unique_weight;
     for lock_attempt in transaction_batch.lock_attempts.iter() {
         let page = lock_attempt.target.page_ref();
-        page.contended_unique_weights
+        page.contended_unique_weights.remove(uq);
     }
     completed_transaction_sender.0
         .send(transaction_batch)

From a352d8ecdd029d21c99a93affa9b56f32270b5c5 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 20 Aug 2022 15:11:46 +0900
Subject: [PATCH 0732/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index b63599d3a609f9..7faefbec0f43a2 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -139,7 +139,7 @@ impl TaskIds {
     }
 
     #[inline(never)]
-    fn remove(&mut self, u: &UniqueWeight) {
+    pub fn remove(&mut self, u: &UniqueWeight) {
         let a = self.task_ids.remove(u);
         /*
         match self.cached_heaviest {

From 280befc77a2530946b6f9b05221648e7f43a3af1 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 20 Aug 2022 15:12:00 +0900
Subject: [PATCH 0733/3199] save

---
 transaction-scheduler-bench/src/main.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/transaction-scheduler-bench/src/main.rs b/transaction-scheduler-bench/src/main.rs
index 4cdb1ad6e006f3..c1e42083b04a7b 100644
--- a/transaction-scheduler-bench/src/main.rs
+++ b/transaction-scheduler-bench/src/main.rs
@@ -347,7 +347,7 @@ fn handle_transaction_batch(
     let uq = transaction_batch.unique_weight;
     for lock_attempt in transaction_batch.lock_attempts.iter() {
         let page = lock_attempt.target.page_ref();
-        page.contended_unique_weights.remove(uq);
+        page.contended_unique_weights.remove(&uq);
     }
     completed_transaction_sender.0
         .send(transaction_batch)

From f0fccf83e1534f532d910b255529a7ada76c70c9 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 20 Aug 2022 15:12:13 +0900
Subject: [PATCH 0734/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 7faefbec0f43a2..cd3f004acb8140 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -139,7 +139,7 @@ impl TaskIds {
     }
 
     #[inline(never)]
-    pub fn remove(&mut self, u: &UniqueWeight) {
+    pub fn remove(&self, u: &UniqueWeight) {
         let a = self.task_ids.remove(u);
         /*
         match self.cached_heaviest {

From 295af3903ba4d29ef42188d71f4a919b3c41885a Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 20 Aug 2022 15:13:31 +0900
Subject: [PATCH 0735/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index cd3f004acb8140..e9422790cfad64 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -289,7 +289,7 @@ impl AddressBook {
     #[inline(never)]
     fn forget_address_contention(unique_weight: &UniqueWeight, a: &mut LockAttempt) {
         if a.remembered {
-            a.target.page_mut().contended_unique_weights.remove(unique_weight);
+            a.target.page().contended_unique_weights.remove(unique_weight);
         }
     }
 

From bd931aba2bb0aaa273edacc37d8bf81830f36930 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 20 Aug 2022 15:13:44 +0900
Subject: [PATCH 0736/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index e9422790cfad64..b636b12ff2a819 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -289,7 +289,7 @@ impl AddressBook {
     #[inline(never)]
     fn forget_address_contention(unique_weight: &UniqueWeight, a: &mut LockAttempt) {
         if a.remembered {
-            a.target.page().contended_unique_weights.remove(unique_weight);
+            a.target.page_ref().contended_unique_weights.remove(unique_weight);
         }
     }
 

From 0191c5178128a602ad4b782fb7f0c4f43bf7f313 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 20 Aug 2022 15:14:14 +0900
Subject: [PATCH 0737/3199] save

---
 scheduler/src/lib.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index b636b12ff2a819..aa298d544c8724 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -727,7 +727,7 @@ impl ScheduleStage {
     ) {
         for mut l in lock_attempts {
             // ensure to remove remaining refs of this unique_weight
-            AddressBook::forget_address_contention(&unique_weight, &mut l);
+            //AddressBook::forget_address_contention(&unique_weight, &mut l);
         }
     }
 
@@ -739,7 +739,7 @@ impl ScheduleStage {
         guaranteed_count: usize,
     ) {
         for mut l in lock_attempts {
-            AddressBook::forget_address_contention(&unique_weight, &mut l);
+            //AddressBook::forget_address_contention(&unique_weight, &mut l);
             match l.status {
                 LockStatus::Guaranteed => {
                     l.target.page_mut().guaranteed_task_ids.insert(*unique_weight, ());

From b8b09e858013292c56f1cb7340be90264df04026 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 20 Aug 2022 15:16:01 +0900
Subject: [PATCH 0738/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index aa298d544c8724..660c37d64c71d7 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -126,7 +126,7 @@ pub struct TaskIds {
 
 impl TaskIds {
     #[inline(never)]
-    fn insert(&mut self, u: UniqueWeight) {
+    fn insert(&self, u: UniqueWeight) {
         /*
         match self.cached_heaviest {
             Some(c) if u > c => { self.cached_heaviest = Some(u) },

From fffb1285fa4fe99bfcdc9402d5fddcf6490c027f Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 20 Aug 2022 15:17:27 +0900
Subject: [PATCH 0739/3199] save

---
 scheduler/src/lib.rs | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 660c37d64c71d7..713d77f9aec8dc 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -126,7 +126,7 @@ pub struct TaskIds {
 
 impl TaskIds {
     #[inline(never)]
-    fn insert(&self, u: UniqueWeight) {
+    fn insert_task_id(&self, u: TaskId) {
         /*
         match self.cached_heaviest {
             Some(c) if u > c => { self.cached_heaviest = Some(u) },
@@ -139,7 +139,7 @@ impl TaskIds {
     }
 
     #[inline(never)]
-    pub fn remove(&self, u: &UniqueWeight) {
+    pub fn remove_task_id(&self, u: &TaskId) {
         let a = self.task_ids.remove(u);
         /*
         match self.cached_heaviest {
@@ -152,12 +152,12 @@ impl TaskIds {
     }
 
     #[inline(never)]
-    fn is_empty(&self) -> bool {
+    fn has_no_task_id(&self) -> bool {
         self.task_ids.is_empty()
     }
 
     #[inline(never)]
-    fn last(&self) -> Option<UniqueWeight> {
+    fn heaviest_task_id(&self) -> Option<TaskId> {
         //self.task_ids.last()
         self.task_ids.back().map(|e| *(e.value()))
         //self.cached_heaviest.as_ref()

From c0eb4b3dfe2c2f44ed4726e3a0b8f8a9b3365ee3 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 20 Aug 2022 15:17:38 +0900
Subject: [PATCH 0740/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 713d77f9aec8dc..17e2ba045bee9e 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -784,7 +784,7 @@ impl ScheduleStage {
 
             let page = l.target.page_mut();
             if newly_uncontended_while_queued && page.next_usage == Usage::Unused {
-                if let Some(uw) = page.contended_unique_weights.last() {
+                if let Some(uw) = page.contended_unique_weights.heaviest_task_id() {
                     address_book.uncontended_task_ids.insert(uw, ());
                 }
             }

From fe63d10b15f93bad7be10be86aacd3941816745f Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 20 Aug 2022 15:17:54 +0900
Subject: [PATCH 0741/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 17e2ba045bee9e..50edea0b095f91 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -343,7 +343,7 @@ impl AddressBook {
 
         if newly_uncontended {
             page.current_usage = Usage::Unused;
-            if !page.contended_unique_weights.is_empty() {
+            if !page.contended_unique_weights.has_no_task_id() {
                 still_queued = true;
             }
         }

From 8e411e20eedd6491a32d04dfcdbaf9f4c5515733 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 20 Aug 2022 15:18:15 +0900
Subject: [PATCH 0742/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 50edea0b095f91..3e343375f66424 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -283,7 +283,7 @@ impl AddressBook {
 
     #[inline(never)]
     fn remember_address_contention(page: &mut Page, unique_weight: &UniqueWeight) {
-        page.contended_unique_weights.insert(*unique_weight);
+        page.contended_unique_weights.insert_task_id(*unique_weight);
     }
 
     #[inline(never)]

From 22becb7fbb1a15566db8ed60ab579b157d94ce10 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 20 Aug 2022 15:18:33 +0900
Subject: [PATCH 0743/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 3e343375f66424..362930433372fd 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -289,7 +289,7 @@ impl AddressBook {
     #[inline(never)]
     fn forget_address_contention(unique_weight: &UniqueWeight, a: &mut LockAttempt) {
         if a.remembered {
-            a.target.page_ref().contended_unique_weights.remove(unique_weight);
+            a.target.page_ref().contended_unique_weights.remove_task_ids(unique_weight);
         }
     }
 

From 5f0dabac2847ad51db6eabaef0d0309aa69619ce Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 20 Aug 2022 15:18:41 +0900
Subject: [PATCH 0744/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 362930433372fd..ade45aee65d35b 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -289,7 +289,7 @@ impl AddressBook {
     #[inline(never)]
     fn forget_address_contention(unique_weight: &UniqueWeight, a: &mut LockAttempt) {
         if a.remembered {
-            a.target.page_ref().contended_unique_weights.remove_task_ids(unique_weight);
+            a.target.page_ref().contended_unique_weights.remove_task_id(unique_weight);
         }
     }
 

From a58bcc0b29547a2664a97f802f69f4cb9613fd28 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 20 Aug 2022 15:18:55 +0900
Subject: [PATCH 0745/3199] save

---
 transaction-scheduler-bench/src/main.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/transaction-scheduler-bench/src/main.rs b/transaction-scheduler-bench/src/main.rs
index c1e42083b04a7b..700d4c9d202969 100644
--- a/transaction-scheduler-bench/src/main.rs
+++ b/transaction-scheduler-bench/src/main.rs
@@ -347,7 +347,7 @@ fn handle_transaction_batch(
     let uq = transaction_batch.unique_weight;
     for lock_attempt in transaction_batch.lock_attempts.iter() {
         let page = lock_attempt.target.page_ref();
-        page.contended_unique_weights.remove(&uq);
+        page.contended_unique_weights.remove_task_id(&uq);
     }
     completed_transaction_sender.0
         .send(transaction_batch)

From 6198bde49d40feed857168991132ac5837299e97 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 20 Aug 2022 15:19:56 +0900
Subject: [PATCH 0746/3199] save

---
 scheduler/src/lib.rs | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index ade45aee65d35b..d9df6200436cc3 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -152,8 +152,8 @@ impl TaskIds {
     }
 
     #[inline(never)]
-    fn has_no_task_id(&self) -> bool {
-        self.task_ids.is_empty()
+    fn has_task_ids(&self) -> bool {
+        !self.task_ids.is_empty()
     }
 
     #[inline(never)]
@@ -343,7 +343,7 @@ impl AddressBook {
 
         if newly_uncontended {
             page.current_usage = Usage::Unused;
-            if !page.contended_unique_weights.has_no_task_id() {
+            if page.contended_unique_weights.has_task_ids() {
                 still_queued = true;
             }
         }

From 0068262ccb067f3ea78fb6b6ab745154dfe962d0 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 20 Aug 2022 15:29:59 +0900
Subject: [PATCH 0747/3199] save

---
 transaction-scheduler-bench/src/main.rs | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/transaction-scheduler-bench/src/main.rs b/transaction-scheduler-bench/src/main.rs
index 700d4c9d202969..e426fc756ec304 100644
--- a/transaction-scheduler-bench/src/main.rs
+++ b/transaction-scheduler-bench/src/main.rs
@@ -455,6 +455,8 @@ fn send_packets(
         for vv in packet_batches {
             for v in vv {
                 let p = solana_scheduler::get_transaction_priority_details(&v.0);
+                for lock_attempt in v.1.iter() {
+                }
                 packet_batch_sender.send(solana_scheduler::Multiplexed::FromPrevious((p, v))).unwrap();
             }
         }

From b6cbd914af4a6649aaec74ee9fe4fabb5c4ef617 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 20 Aug 2022 15:30:21 +0900
Subject: [PATCH 0748/3199] save

---
 transaction-scheduler-bench/src/main.rs | 1 +
 1 file changed, 1 insertion(+)

diff --git a/transaction-scheduler-bench/src/main.rs b/transaction-scheduler-bench/src/main.rs
index e426fc756ec304..a9692b9f7c479d 100644
--- a/transaction-scheduler-bench/src/main.rs
+++ b/transaction-scheduler-bench/src/main.rs
@@ -456,6 +456,7 @@ fn send_packets(
             for v in vv {
                 let p = solana_scheduler::get_transaction_priority_details(&v.0);
                 for lock_attempt in v.1.iter() {
+                    lock_attempt.target.page_ref()
                 }
                 packet_batch_sender.send(solana_scheduler::Multiplexed::FromPrevious((p, v))).unwrap();
             }

From e189d6626b135ea57bc37945f2187863fef84055 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 20 Aug 2022 15:30:54 +0900
Subject: [PATCH 0749/3199] save

---
 transaction-scheduler-bench/src/main.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/transaction-scheduler-bench/src/main.rs b/transaction-scheduler-bench/src/main.rs
index a9692b9f7c479d..ba9aec4af73e89 100644
--- a/transaction-scheduler-bench/src/main.rs
+++ b/transaction-scheduler-bench/src/main.rs
@@ -456,7 +456,7 @@ fn send_packets(
             for v in vv {
                 let p = solana_scheduler::get_transaction_priority_details(&v.0);
                 for lock_attempt in v.1.iter() {
-                    lock_attempt.target.page_ref()
+                    lock_attempt.target.page_ref().contended_unique_weights.insert_task_id(p);
                 }
                 packet_batch_sender.send(solana_scheduler::Multiplexed::FromPrevious((p, v))).unwrap();
             }

From 8d1c49cd88d1ea0ff64ddc06046d64ea01aec1ec Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 20 Aug 2022 15:31:07 +0900
Subject: [PATCH 0750/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index d9df6200436cc3..46cf0a6a44ebfd 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -126,7 +126,7 @@ pub struct TaskIds {
 
 impl TaskIds {
     #[inline(never)]
-    fn insert_task_id(&self, u: TaskId) {
+    pub fn insert_task_id(&self, u: TaskId) {
         /*
         match self.cached_heaviest {
             Some(c) if u > c => { self.cached_heaviest = Some(u) },

From 7946e41cc899cbddbee78582ace5e42ad4f1d85a Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 20 Aug 2022 15:35:09 +0900
Subject: [PATCH 0751/3199] save

---
 scheduler/src/lib.rs                    | 3 ++-
 transaction-scheduler-bench/src/main.rs | 2 ++
 2 files changed, 4 insertions(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 46cf0a6a44ebfd..a644b4276a7bea 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -533,7 +533,8 @@ impl ScheduleStage {
         //    )
         //    .unwrap();
         //tx.foo();
-        let unique_weight = (weight << 32) | (*unique_key & 0x0000_0000_ffff_ffff);
+        //let unique_weight = (weight << 32) | (*unique_key & 0x0000_0000_ffff_ffff);
+        let unique_weight = weight;
 
         runnable_queue.add_to_schedule(
             /*
diff --git a/transaction-scheduler-bench/src/main.rs b/transaction-scheduler-bench/src/main.rs
index ba9aec4af73e89..619ccff2bf2e2f 100644
--- a/transaction-scheduler-bench/src/main.rs
+++ b/transaction-scheduler-bench/src/main.rs
@@ -451,10 +451,12 @@ fn send_packets(
             packet_batches.iter().map(|pb| pb.len()).sum(),
             Ordering::Relaxed,
         );
+        let mut rng = rand::thread_rng();
 
         for vv in packet_batches {
             for v in vv {
                 let p = solana_scheduler::get_transaction_priority_details(&v.0);
+                let p = (p << 32) | (rng.gen::<u64>() & 0x0000_0000_ffff_ffff);
                 for lock_attempt in v.1.iter() {
                     lock_attempt.target.page_ref().contended_unique_weights.insert_task_id(p);
                 }

From 83be1d6a929edca3701323beeb6fc8899018ba15 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 20 Aug 2022 15:38:04 +0900
Subject: [PATCH 0752/3199] save

---
 scheduler/src/lib.rs | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index a644b4276a7bea..2e237930615659 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -162,6 +162,13 @@ impl TaskIds {
         self.task_ids.back().map(|e| *(e.value()))
         //self.cached_heaviest.as_ref()
     }
+
+    #[inline(never)]
+    fn heaviest_task_cursor(&self) -> Option<TaskId> {
+        //self.task_ids.last()
+        self.task_ids.back()
+        //self.cached_heaviest.as_ref()
+    }
 }
 
 #[derive(Debug)]
@@ -785,7 +792,7 @@ impl ScheduleStage {
 
             let page = l.target.page_mut();
             if newly_uncontended_while_queued && page.next_usage == Usage::Unused {
-                if let Some(uw) = page.contended_unique_weights.heaviest_task_id() {
+                if let Some(uw) = page.contended_unique_weights.heaviest_task_cursor() {
                     address_book.uncontended_task_ids.insert(uw, ());
                 }
             }

From 8286fba73f3ad05c3c7a04b64388f7d21953ac40 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 20 Aug 2022 15:38:34 +0900
Subject: [PATCH 0753/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 2e237930615659..6afa6a9996bcb7 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -164,7 +164,7 @@ impl TaskIds {
     }
 
     #[inline(never)]
-    fn heaviest_task_cursor(&self) -> Option<TaskId> {
+    fn heaviest_task_cursor(&self) -> Option<crossbeam_skiplist::set::Entry<'_, u64>> {
         //self.task_ids.last()
         self.task_ids.back()
         //self.cached_heaviest.as_ref()

From a1daa4315c86cd4c073aa0fd18f73d653f9083d2 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 20 Aug 2022 15:44:48 +0900
Subject: [PATCH 0754/3199] save

---
 scheduler/src/lib.rs | 15 +++++++++++++--
 1 file changed, 13 insertions(+), 2 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 6afa6a9996bcb7..a9b47bbf5895da 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -792,8 +792,19 @@ impl ScheduleStage {
 
             let page = l.target.page_mut();
             if newly_uncontended_while_queued && page.next_usage == Usage::Unused {
-                if let Some(uw) = page.contended_unique_weights.heaviest_task_cursor() {
-                    address_book.uncontended_task_ids.insert(uw, ());
+                if let Some(cursor) = page.contended_unique_weights.heaviest_task_cursor() {
+                    let mut found = true;
+                    while !contended_queue.has_task(cursor.value()) {
+                        if let Some(new_cursor) = cursor.prev() {
+                            cursor = new_cursor;
+                        } else {
+                            found = false;
+                            break;
+                        }
+                    }
+                    if found {
+                        address_book.uncontended_task_ids.insert(uw, ());
+                    }
                 }
             }
             if page.current_usage == Usage::Unused && page.next_usage != Usage::Unused {

From b02f6741c8f89e93bb354f46df91c5ea76e9af03 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 20 Aug 2022 15:45:07 +0900
Subject: [PATCH 0755/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index a9b47bbf5895da..41443b839d0e12 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -803,7 +803,7 @@ impl ScheduleStage {
                         }
                     }
                     if found {
-                        address_book.uncontended_task_ids.insert(uw, ());
+                        address_book.uncontended_task_ids.insert(cursor.value(), ());
                     }
                 }
             }

From 23170992442d83732dc339e5eac0ab651f49667a Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 20 Aug 2022 15:45:21 +0900
Subject: [PATCH 0756/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 41443b839d0e12..ca739f628ea3cf 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -803,7 +803,7 @@ impl ScheduleStage {
                         }
                     }
                     if found {
-                        address_book.uncontended_task_ids.insert(cursor.value(), ());
+                        address_book.uncontended_task_ids.insert(*(cursor.value()), ());
                     }
                 }
             }

From e343a09a3cf9695661f9bc10ddd5fb212ab30433 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 20 Aug 2022 15:46:19 +0900
Subject: [PATCH 0757/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index ca739f628ea3cf..83b5aa36f8a74c 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -786,7 +786,7 @@ impl ScheduleStage {
     }
 
     #[inline(never)]
-    fn unlock_after_execution(address_book: &mut AddressBook, lock_attempts: Vec<LockAttempt>) {
+    fn unlock_after_execution(address_book: &mut AddressBook, contended_queue: &TaskQueue, lock_attempts: Vec<LockAttempt>) {
         for mut l in lock_attempts.into_iter() {
             let newly_uncontended_while_queued = address_book.reset_lock(&mut l, true);
 

From 0d5dec93e3fdba3d9b950d388ddd5f4f12e702b9 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 20 Aug 2022 15:47:01 +0900
Subject: [PATCH 0758/3199] save

---
 scheduler/src/lib.rs | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 83b5aa36f8a74c..52dd57716beabc 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -852,10 +852,10 @@ impl ScheduleStage {
     }
 
     #[inline(never)]
-    fn commit_result(ee: &mut ExecutionEnvironment, address_book: &mut AddressBook) {
+    fn commit_result(ee: &mut ExecutionEnvironment, address_book: &mut AddressBook, contended_queue: &ContendedQueue) {
         let lock_attempts = std::mem::take(&mut ee.lock_attempts);
         // do par()-ly?
-        Self::unlock_after_execution(address_book, lock_attempts);
+        Self::unlock_after_execution(address_book, contended_queue, lock_attempts);
         // block-wide qos validation will be done here
         // if error risen..:
         //   don't commit the tx for banking and potentially finish scheduling at block max cu
@@ -915,7 +915,7 @@ impl ScheduleStage {
                                 trace!("recv from execute: {:?}", processed_execution_environment.unique_weight);
                                 executing_queue_count -= 1;
 
-                                Self::commit_result(&mut processed_execution_environment, address_book);
+                                Self::commit_result(&mut processed_execution_environment, address_book, contended_queue);
                                 // async-ly propagate the result to rpc subsystems
                                 if let Some(to_next_stage) = to_next_stage {
                                     to_next_stage.send(processed_execution_environment).unwrap();

From d94e8c8c1c5533c839650fc504cf73742457fd5c Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 20 Aug 2022 15:47:17 +0900
Subject: [PATCH 0759/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 52dd57716beabc..a015075a2756f3 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -948,7 +948,7 @@ impl ScheduleStage {
                         trace!("recv from execute: {:?}", processed_execution_environment.unique_weight);
                         executing_queue_count -= 1;
 
-                        Self::commit_result(&mut processed_execution_environment, address_book);
+                        Self::commit_result(&mut processed_execution_environment, address_book, contended_queue);
                         // async-ly propagate the result to rpc subsystems
                         if let Some(to_next_stage) = to_next_stage {
                             to_next_stage.send(processed_execution_environment).unwrap();

From 020cd7edf47ee0de021b7c5d7829450e8c3051cc Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 20 Aug 2022 15:47:30 +0900
Subject: [PATCH 0760/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index a015075a2756f3..128e767321cd20 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -852,7 +852,7 @@ impl ScheduleStage {
     }
 
     #[inline(never)]
-    fn commit_result(ee: &mut ExecutionEnvironment, address_book: &mut AddressBook, contended_queue: &ContendedQueue) {
+    fn commit_result(ee: &mut ExecutionEnvironment, address_book: &mut AddressBook, contended_queue: &TaskQueue) {
         let lock_attempts = std::mem::take(&mut ee.lock_attempts);
         // do par()-ly?
         Self::unlock_after_execution(address_book, contended_queue, lock_attempts);

From 4911f74f99719fb282ca52b3be40a10986210ba4 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 20 Aug 2022 15:48:22 +0900
Subject: [PATCH 0761/3199] save

---
 scheduler/src/lib.rs | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 128e767321cd20..92a17884d2bb64 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -459,6 +459,10 @@ impl TaskQueue {
         }
     }
 
+    fn has_task(&self, unique_weight: &UniqueWeight) -> bool {
+        self.tasks.has_key(unique_weight)
+    }
+
     #[inline(never)]
     fn heaviest_entry_to_execute(
         &mut self,

From 9a6ffb547775bed0be70dedf3c340177c5118967 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 20 Aug 2022 15:48:34 +0900
Subject: [PATCH 0762/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 92a17884d2bb64..076127685cde98 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -460,7 +460,7 @@ impl TaskQueue {
     }
 
     fn has_task(&self, unique_weight: &UniqueWeight) -> bool {
-        self.tasks.has_key(unique_weight)
+        self.tasks.includes_key(unique_weight)
     }
 
     #[inline(never)]

From eab9bd27d887110797afe1d3e27f81f1809aae49 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 20 Aug 2022 15:48:43 +0900
Subject: [PATCH 0763/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 076127685cde98..5de94382361481 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -460,7 +460,7 @@ impl TaskQueue {
     }
 
     fn has_task(&self, unique_weight: &UniqueWeight) -> bool {
-        self.tasks.includes_key(unique_weight)
+        self.tasks.contains_key(unique_weight)
     }
 
     #[inline(never)]

From 9a5013a606bca2fc1a61b61e2f6c2898f44c9322 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 20 Aug 2022 15:52:15 +0900
Subject: [PATCH 0764/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 5de94382361481..2895e90a6f9604 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -796,7 +796,7 @@ impl ScheduleStage {
 
             let page = l.target.page_mut();
             if newly_uncontended_while_queued && page.next_usage == Usage::Unused {
-                if let Some(cursor) = page.contended_unique_weights.heaviest_task_cursor() {
+                if let Some(mut cursor) = page.contended_unique_weights.heaviest_task_cursor() {
                     let mut found = true;
                     while !contended_queue.has_task(cursor.value()) {
                         if let Some(new_cursor) = cursor.prev() {

From 762ed0fce6287b829e07dce51210e119858a1c27 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 20 Aug 2022 15:52:38 +0900
Subject: [PATCH 0765/3199] save

---
 scheduler/src/lib.rs | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 2895e90a6f9604..78eb25bce7faee 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -796,18 +796,18 @@ impl ScheduleStage {
 
             let page = l.target.page_mut();
             if newly_uncontended_while_queued && page.next_usage == Usage::Unused {
-                if let Some(mut cursor) = page.contended_unique_weights.heaviest_task_cursor() {
+                if let Some(mut task_cursor) = page.contended_unique_weights.heaviest_task_cursor() {
                     let mut found = true;
-                    while !contended_queue.has_task(cursor.value()) {
-                        if let Some(new_cursor) = cursor.prev() {
-                            cursor = new_cursor;
+                    while !contended_queue.has_task(task_cursor.value()) {
+                        if let Some(new_cursor) = task_cursor.prev() {
+                            task_cursor = new_cursor;
                         } else {
                             found = false;
                             break;
                         }
                     }
                     if found {
-                        address_book.uncontended_task_ids.insert(*(cursor.value()), ());
+                        address_book.uncontended_task_ids.insert(*(task_cursor.value()), ());
                     }
                 }
             }

From 32cfcf459f4ace1c3df9efcc0cd3dc1718c29462 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 20 Aug 2022 15:55:14 +0900
Subject: [PATCH 0766/3199] save

---
 scheduler/src/lib.rs | 12 +++++++-----
 1 file changed, 7 insertions(+), 5 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 78eb25bce7faee..43d799ac5881ad 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -72,7 +72,7 @@ pub struct LockAttempt {
     pub target: PageRc,
     status: LockStatus,
     requested_usage: RequestedUsage,
-    remembered: bool,
+    //remembered: bool,
 }
 
 impl LockAttempt {
@@ -81,7 +81,7 @@ impl LockAttempt {
             target,
             status: LockStatus::Succeded,
             requested_usage,
-            remembered: false,
+            //remembered: false,
         }
     }
 }
@@ -222,7 +222,7 @@ impl AddressBook {
         unique_weight: &UniqueWeight,
         attempt: &mut LockAttempt,
     ) {
-        let LockAttempt {target, requested_usage, status, remembered} = attempt;
+        let LockAttempt {target, requested_usage, status/*, remembered*/} = attempt;
 
                 let mut page = target.page_mut();
 
@@ -288,6 +288,7 @@ impl AddressBook {
                 }
     }
 
+    /*
     #[inline(never)]
     fn remember_address_contention(page: &mut Page, unique_weight: &UniqueWeight) {
         page.contended_unique_weights.insert_task_id(*unique_weight);
@@ -299,6 +300,7 @@ impl AddressBook {
             a.target.page_ref().contended_unique_weights.remove_task_id(unique_weight);
         }
     }
+    */
 
     fn reset_lock(&mut self, attempt: &mut LockAttempt, after_execution: bool) -> bool {
         match attempt.status {
@@ -777,8 +779,8 @@ impl ScheduleStage {
     ) {
         for l in lock_attempts {
             if from_runnable {
-                AddressBook::remember_address_contention(&mut l.target.page_mut(), unique_weight);
-                l.remembered = true;
+                //AddressBook::remember_address_contention(&mut l.target.page_mut(), unique_weight);
+                //l.remembered = true;
             }
             address_book.reset_lock(l, false);
             //if let Some(uw) = l.target.page().contended_unique_weights.last() {

From 8f7eaa0abcd5d219c2e8794ccb322d0461d7ca2f Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 20 Aug 2022 16:26:26 +0900
Subject: [PATCH 0767/3199] save

---
 transaction-scheduler-bench/src/main.rs | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/transaction-scheduler-bench/src/main.rs b/transaction-scheduler-bench/src/main.rs
index 619ccff2bf2e2f..f29b068a4bdaaa 100644
--- a/transaction-scheduler-bench/src/main.rs
+++ b/transaction-scheduler-bench/src/main.rs
@@ -346,8 +346,7 @@ fn handle_transaction_batch(
 
     let uq = transaction_batch.unique_weight;
     for lock_attempt in transaction_batch.lock_attempts.iter() {
-        let page = lock_attempt.target.page_ref();
-        page.contended_unique_weights.remove_task_id(&uq);
+        lock_attempt.target.page_ref().contended_unique_weights.remove_task_id(&uq);
     }
     completed_transaction_sender.0
         .send(transaction_batch)

From bae202a9388e30aa8bc2f491e0ad20aca7116d44 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 20 Aug 2022 21:41:07 +0900
Subject: [PATCH 0768/3199] save

---
 scheduler/src/lib.rs | 48 ++++++++++++++++++++++----------------------
 1 file changed, 24 insertions(+), 24 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 43d799ac5881ad..99fc42b0dd74b5 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -176,7 +176,7 @@ pub struct Page {
     current_usage: Usage,
     next_usage: Usage,
     pub contended_unique_weights: TaskIds,
-    guaranteed_task_ids: WeightedTaskIds,
+    provional_task_ids: WeightedTaskIds,
     //loaded account from Accounts db
     //comulative_cu for qos; i.e. track serialized cumulative keyed by addresses and bail out block
     //producing as soon as any one of cu from the executing thread reaches to the limit
@@ -188,7 +188,7 @@ impl Page {
             current_usage,
             next_usage: Usage::Unused,
             contended_unique_weights: Default::default(),
-            guaranteed_task_ids: Default::default(),
+            provional_task_ids: Default::default(),
         }
     }
 
@@ -210,7 +210,7 @@ type AddressMapEntry<'a> = dashmap::mapref::entry::Entry<'a, Pubkey, PageRc>;
 pub struct AddressBook {
     book: AddressMap,
     uncontended_task_ids: WeightedTaskIds,
-    runnable_guaranteed_task_ids: WeightedTaskIds,
+    runnable_provional_task_ids: WeightedTaskIds,
     gurantee_timers: std::collections::HashMap<UniqueWeight, usize>, 
 }
 
@@ -488,7 +488,7 @@ fn attempt_lock_for_execution<'a>(
 ) -> (usize, usize, Vec<LockAttempt>) {
     // no short-cuircuit; we at least all need to add to the contended queue
     let mut unlockable_count = 0;
-    let mut guaranteed_count = 0;
+    let mut provional_count = 0;
 
     for attempt in placeholder_attempts.iter_mut() {
         AddressBook::attempt_lock_address(from_runnable, prefer_immediate, unique_weight, attempt);
@@ -498,12 +498,12 @@ fn attempt_lock_for_execution<'a>(
                 unlockable_count += 1;
             },
             LockStatus::Guaranteed => {
-                guaranteed_count += 1;
+                provional_count += 1;
             },
         }
     }
 
-    (unlockable_count, guaranteed_count, placeholder_attempts)
+    (unlockable_count, provional_count, placeholder_attempts)
 }
 
 type PreprocessedTransaction = (SanitizedTransaction, Vec<LockAttempt>);
@@ -642,8 +642,8 @@ impl ScheduleStage {
         address_book: &mut AddressBook,
         prefer_immediate: bool,
     ) -> Option<(UniqueWeight, Task, Vec<LockAttempt>)> {
-        if let Some(a) = address_book.runnable_guaranteed_task_ids.pop_last() {
-            trace!("expediate pop from guaranteed queue [rest: {}]", address_book.runnable_guaranteed_task_ids.len());
+        if let Some(a) = address_book.runnable_provional_task_ids.pop_last() {
+            trace!("expediate pop from provisional queue [rest: {}]", address_book.runnable_provional_task_ids.len());
             let queue_entry = contended_queue.entry_to_execute(a.0);
             let mut task = queue_entry.remove();
             let ll = std::mem::take(&mut task.tx.1);
@@ -663,7 +663,7 @@ impl ScheduleStage {
             // plumb message_hash into StatusCache or implmenent our own for duplicate tx
             // detection?
 
-            let (unlockable_count, guaranteed_count, mut populated_lock_attempts) = attempt_lock_for_execution(
+            let (unlockable_count, provional_count, mut populated_lock_attempts) = attempt_lock_for_execution(
                 from_runnable,
                 prefer_immediate,
                 address_book,
@@ -685,13 +685,13 @@ impl ScheduleStage {
                 next_task.contention_count += 1;
 
                 if from_runnable {
-                    trace!("move to contended due to lock failure [{}/{}/{}]", unlockable_count, guaranteed_count, lock_count);
+                    trace!("move to contended due to lock failure [{}/{}/{}]", unlockable_count, provional_count, lock_count);
                     reborrowed_contended_queue
                         .unwrap()
                         .add_to_schedule(*queue_entry.key(), queue_entry.remove());
                     // maybe run lightweight prune logic on contended_queue here.
                 } else {
-                    trace!("relock failed [{}/{}/{}]; remains in contended: {:?} contention: {}", unlockable_count, guaranteed_count, lock_count, &unique_weight, next_task.contention_count);
+                    trace!("relock failed [{}/{}/{}]; remains in contended: {:?} contention: {}", unlockable_count, provional_count, lock_count, &unique_weight, next_task.contention_count);
                     //address_book.uncontended_task_ids.clear();
                 }
 
@@ -700,15 +700,15 @@ impl ScheduleStage {
                 } else {
                     return None;
                 }
-            } else if guaranteed_count > 0 {
+            } else if provional_count > 0 {
                 assert!(!from_runnable);
                 let lock_count = populated_lock_attempts.len();
-                trace!("guaranteed exec: [{}/{}]", guaranteed_count, lock_count);
-                Self::finalize_lock_for_guaranteed_execution(
+                trace!("provisional exec: [{}/{}]", provional_count, lock_count);
+                Self::finalize_lock_for_provional_execution(
                     address_book,
                     &unique_weight,
                     &mut populated_lock_attempts,
-                    guaranteed_count,
+                    provional_count,
                 );
                 std::mem::swap(&mut next_task.tx.1, &mut populated_lock_attempts);
 
@@ -746,17 +746,17 @@ impl ScheduleStage {
     }
 
     #[inline(never)]
-    fn finalize_lock_for_guaranteed_execution(
+    fn finalize_lock_for_provional_execution(
         address_book: &mut AddressBook,
         unique_weight: &UniqueWeight,
         lock_attempts: &mut Vec<LockAttempt>,
-        guaranteed_count: usize,
+        provional_count: usize,
     ) {
         for mut l in lock_attempts {
             //AddressBook::forget_address_contention(&unique_weight, &mut l);
             match l.status {
                 LockStatus::Guaranteed => {
-                    l.target.page_mut().guaranteed_task_ids.insert(*unique_weight, ());
+                    l.target.page_mut().provional_task_ids.insert(*unique_weight, ());
                 }
                 LockStatus::Succeded => {
                     // do nothing
@@ -766,7 +766,7 @@ impl ScheduleStage {
                 }
             }
         }
-        address_book.gurantee_timers.insert(*unique_weight, guaranteed_count);
+        address_book.gurantee_timers.insert(*unique_weight, provional_count);
         trace!("gurantee_timers: {}", address_book.gurantee_timers.len());
     }
 
@@ -815,17 +815,17 @@ impl ScheduleStage {
             }
             if page.current_usage == Usage::Unused && page.next_usage != Usage::Unused {
                 page.switch_to_next_usage();
-                for task_id in std::mem::take(&mut page.guaranteed_task_ids).keys() {
+                for task_id in std::mem::take(&mut page.provional_task_ids).keys() {
                     match address_book.gurantee_timers.entry(*task_id) {
                         std::collections::hash_map::Entry::Occupied(mut timer_entry) => {
                             let count = timer_entry.get_mut();
                             *count = count.checked_sub(1).unwrap();
                             if *count == 0 {
-                                trace!("guaranteed lock decrease: {} => {} (!)", *count + 1, *count);
+                                trace!("provisional lock decrease: {} => {} (!)", *count + 1, *count);
                                 timer_entry.remove();
-                                address_book.runnable_guaranteed_task_ids.insert(*task_id, ());
+                                address_book.runnable_provional_task_ids.insert(*task_id, ());
                             } else {
-                                trace!("guaranteed lock decrease: {} => {}", *count + 1, *count);
+                                trace!("provisional lock decrease: {} => {}", *count + 1, *count);
                             }
                         },
                         std::collections::hash_map::Entry::Vacant(_) => {
@@ -907,7 +907,7 @@ impl ScheduleStage {
         let mut current_unique_key = u64::max_value();
 
         loop {
-            trace!("schedule_once (from: {}, to: {}, runnnable: {}, contended: {}, (immediate+guaranteed)/max: ({}+{})/{}) active from contended: {}!", from.len(), to_execute_substage.len(), runnable_queue.task_count(), contended_queue.task_count(), executing_queue_count, address_book.gurantee_timers.len(), max_executing_queue_count, address_book.uncontended_task_ids.len());
+            trace!("schedule_once (from: {}, to: {}, runnnable: {}, contended: {}, (immediate+provisional)/max: ({}+{})/{}) active from contended: {}!", from.len(), to_execute_substage.len(), runnable_queue.task_count(), contended_queue.task_count(), executing_queue_count, address_book.gurantee_timers.len(), max_executing_queue_count, address_book.uncontended_task_ids.len());
 
             crossbeam_channel::select! {
                recv(from) -> maybe_from => {

From 449d2170f89e35cc2526ae7c0c71ac945231358f Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 20 Aug 2022 21:42:59 +0900
Subject: [PATCH 0769/3199] save

---
 scheduler/src/lib.rs | 42 +++++++++++++++++++++---------------------
 1 file changed, 21 insertions(+), 21 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 99fc42b0dd74b5..b9c2344c0f54ba 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -176,7 +176,7 @@ pub struct Page {
     current_usage: Usage,
     next_usage: Usage,
     pub contended_unique_weights: TaskIds,
-    provional_task_ids: WeightedTaskIds,
+    provisional_task_ids: WeightedTaskIds,
     //loaded account from Accounts db
     //comulative_cu for qos; i.e. track serialized cumulative keyed by addresses and bail out block
     //producing as soon as any one of cu from the executing thread reaches to the limit
@@ -188,7 +188,7 @@ impl Page {
             current_usage,
             next_usage: Usage::Unused,
             contended_unique_weights: Default::default(),
-            provional_task_ids: Default::default(),
+            provisional_task_ids: Default::default(),
         }
     }
 
@@ -210,7 +210,7 @@ type AddressMapEntry<'a> = dashmap::mapref::entry::Entry<'a, Pubkey, PageRc>;
 pub struct AddressBook {
     book: AddressMap,
     uncontended_task_ids: WeightedTaskIds,
-    runnable_provional_task_ids: WeightedTaskIds,
+    runnable_provisional_task_ids: WeightedTaskIds,
     gurantee_timers: std::collections::HashMap<UniqueWeight, usize>, 
 }
 
@@ -488,7 +488,7 @@ fn attempt_lock_for_execution<'a>(
 ) -> (usize, usize, Vec<LockAttempt>) {
     // no short-cuircuit; we at least all need to add to the contended queue
     let mut unlockable_count = 0;
-    let mut provional_count = 0;
+    let mut provisional_count = 0;
 
     for attempt in placeholder_attempts.iter_mut() {
         AddressBook::attempt_lock_address(from_runnable, prefer_immediate, unique_weight, attempt);
@@ -498,12 +498,12 @@ fn attempt_lock_for_execution<'a>(
                 unlockable_count += 1;
             },
             LockStatus::Guaranteed => {
-                provional_count += 1;
+                provisional_count += 1;
             },
         }
     }
 
-    (unlockable_count, provional_count, placeholder_attempts)
+    (unlockable_count, provisional_count, placeholder_attempts)
 }
 
 type PreprocessedTransaction = (SanitizedTransaction, Vec<LockAttempt>);
@@ -642,8 +642,8 @@ impl ScheduleStage {
         address_book: &mut AddressBook,
         prefer_immediate: bool,
     ) -> Option<(UniqueWeight, Task, Vec<LockAttempt>)> {
-        if let Some(a) = address_book.runnable_provional_task_ids.pop_last() {
-            trace!("expediate pop from provisional queue [rest: {}]", address_book.runnable_provional_task_ids.len());
+        if let Some(a) = address_book.runnable_provisional_task_ids.pop_last() {
+            trace!("expediate pop from provisional queue [rest: {}]", address_book.runnable_provisional_task_ids.len());
             let queue_entry = contended_queue.entry_to_execute(a.0);
             let mut task = queue_entry.remove();
             let ll = std::mem::take(&mut task.tx.1);
@@ -663,7 +663,7 @@ impl ScheduleStage {
             // plumb message_hash into StatusCache or implmenent our own for duplicate tx
             // detection?
 
-            let (unlockable_count, provional_count, mut populated_lock_attempts) = attempt_lock_for_execution(
+            let (unlockable_count, provisional_count, mut populated_lock_attempts) = attempt_lock_for_execution(
                 from_runnable,
                 prefer_immediate,
                 address_book,
@@ -685,13 +685,13 @@ impl ScheduleStage {
                 next_task.contention_count += 1;
 
                 if from_runnable {
-                    trace!("move to contended due to lock failure [{}/{}/{}]", unlockable_count, provional_count, lock_count);
+                    trace!("move to contended due to lock failure [{}/{}/{}]", unlockable_count, provisional_count, lock_count);
                     reborrowed_contended_queue
                         .unwrap()
                         .add_to_schedule(*queue_entry.key(), queue_entry.remove());
                     // maybe run lightweight prune logic on contended_queue here.
                 } else {
-                    trace!("relock failed [{}/{}/{}]; remains in contended: {:?} contention: {}", unlockable_count, provional_count, lock_count, &unique_weight, next_task.contention_count);
+                    trace!("relock failed [{}/{}/{}]; remains in contended: {:?} contention: {}", unlockable_count, provisional_count, lock_count, &unique_weight, next_task.contention_count);
                     //address_book.uncontended_task_ids.clear();
                 }
 
@@ -700,15 +700,15 @@ impl ScheduleStage {
                 } else {
                     return None;
                 }
-            } else if provional_count > 0 {
+            } else if provisional_count > 0 {
                 assert!(!from_runnable);
                 let lock_count = populated_lock_attempts.len();
-                trace!("provisional exec: [{}/{}]", provional_count, lock_count);
-                Self::finalize_lock_for_provional_execution(
+                trace!("provisional exec: [{}/{}]", provisional_count, lock_count);
+                Self::finalize_lock_for_provisional_execution(
                     address_book,
                     &unique_weight,
                     &mut populated_lock_attempts,
-                    provional_count,
+                    provisional_count,
                 );
                 std::mem::swap(&mut next_task.tx.1, &mut populated_lock_attempts);
 
@@ -746,17 +746,17 @@ impl ScheduleStage {
     }
 
     #[inline(never)]
-    fn finalize_lock_for_provional_execution(
+    fn finalize_lock_for_provisional_execution(
         address_book: &mut AddressBook,
         unique_weight: &UniqueWeight,
         lock_attempts: &mut Vec<LockAttempt>,
-        provional_count: usize,
+        provisional_count: usize,
     ) {
         for mut l in lock_attempts {
             //AddressBook::forget_address_contention(&unique_weight, &mut l);
             match l.status {
                 LockStatus::Guaranteed => {
-                    l.target.page_mut().provional_task_ids.insert(*unique_weight, ());
+                    l.target.page_mut().provisional_task_ids.insert(*unique_weight, ());
                 }
                 LockStatus::Succeded => {
                     // do nothing
@@ -766,7 +766,7 @@ impl ScheduleStage {
                 }
             }
         }
-        address_book.gurantee_timers.insert(*unique_weight, provional_count);
+        address_book.gurantee_timers.insert(*unique_weight, provisional_count);
         trace!("gurantee_timers: {}", address_book.gurantee_timers.len());
     }
 
@@ -815,7 +815,7 @@ impl ScheduleStage {
             }
             if page.current_usage == Usage::Unused && page.next_usage != Usage::Unused {
                 page.switch_to_next_usage();
-                for task_id in std::mem::take(&mut page.provional_task_ids).keys() {
+                for task_id in std::mem::take(&mut page.provisional_task_ids).keys() {
                     match address_book.gurantee_timers.entry(*task_id) {
                         std::collections::hash_map::Entry::Occupied(mut timer_entry) => {
                             let count = timer_entry.get_mut();
@@ -823,7 +823,7 @@ impl ScheduleStage {
                             if *count == 0 {
                                 trace!("provisional lock decrease: {} => {} (!)", *count + 1, *count);
                                 timer_entry.remove();
-                                address_book.runnable_provional_task_ids.insert(*task_id, ());
+                                address_book.runnable_provisional_task_ids.insert(*task_id, ());
                             } else {
                                 trace!("provisional lock decrease: {} => {}", *count + 1, *count);
                             }

From 3db635d84f70bf605de32596f635f15e57652cd8 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 20 Aug 2022 22:04:20 +0900
Subject: [PATCH 0770/3199] save

---
 scheduler/src/lib.rs | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index b9c2344c0f54ba..e18da2dd90eba3 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -211,7 +211,7 @@ pub struct AddressBook {
     book: AddressMap,
     uncontended_task_ids: WeightedTaskIds,
     runnable_provisional_task_ids: WeightedTaskIds,
-    gurantee_timers: std::collections::HashMap<UniqueWeight, usize>, 
+    provisioning_trackers: std::collections::HashMap<UniqueWeight, usize>, 
 }
 
 impl AddressBook {
@@ -766,8 +766,8 @@ impl ScheduleStage {
                 }
             }
         }
-        address_book.gurantee_timers.insert(*unique_weight, provisional_count);
-        trace!("gurantee_timers: {}", address_book.gurantee_timers.len());
+        address_book.provisioning_trackers.insert(*unique_weight, provisional_count);
+        trace!("provisioning_trackers: {}", address_book.provisioning_trackers.len());
     }
 
     #[inline(never)]
@@ -816,7 +816,7 @@ impl ScheduleStage {
             if page.current_usage == Usage::Unused && page.next_usage != Usage::Unused {
                 page.switch_to_next_usage();
                 for task_id in std::mem::take(&mut page.provisional_task_ids).keys() {
-                    match address_book.gurantee_timers.entry(*task_id) {
+                    match address_book.provisioning_trackers.entry(*task_id) {
                         std::collections::hash_map::Entry::Occupied(mut timer_entry) => {
                             let count = timer_entry.get_mut();
                             *count = count.checked_sub(1).unwrap();
@@ -907,7 +907,7 @@ impl ScheduleStage {
         let mut current_unique_key = u64::max_value();
 
         loop {
-            trace!("schedule_once (from: {}, to: {}, runnnable: {}, contended: {}, (immediate+provisional)/max: ({}+{})/{}) active from contended: {}!", from.len(), to_execute_substage.len(), runnable_queue.task_count(), contended_queue.task_count(), executing_queue_count, address_book.gurantee_timers.len(), max_executing_queue_count, address_book.uncontended_task_ids.len());
+            trace!("schedule_once (from: {}, to: {}, runnnable: {}, contended: {}, (immediate+provisional)/max: ({}+{})/{}) active from contended: {}!", from.len(), to_execute_substage.len(), runnable_queue.task_count(), contended_queue.task_count(), executing_queue_count, address_book.provisioning_trackers.len(), max_executing_queue_count, address_book.uncontended_task_ids.len());
 
             crossbeam_channel::select! {
                recv(from) -> maybe_from => {
@@ -971,7 +971,7 @@ impl ScheduleStage {
             loop {
                 /* if !address_book.uncontended_task_ids.is_empty() {
                     trace!("prefer emptying n_u_a");
-                } else */ if (executing_queue_count + address_book.gurantee_timers.len()) >= max_executing_queue_count {
+                } else */ if (executing_queue_count + address_book.provisioning_trackers.len()) >= max_executing_queue_count {
                     trace!("skip scheduling; outgoing queue full");
                     while from.len() > 0 {
                        let i = from.recv().unwrap();
@@ -988,7 +988,7 @@ impl ScheduleStage {
                     break;
                 }
 
-                let prefer_immediate = address_book.gurantee_timers.len()/4 > executing_queue_count;
+                let prefer_immediate = address_book.provisioning_trackers.len()/4 > executing_queue_count;
                 let maybe_ee =
                     Self::schedule_next_execution(runnable_queue, contended_queue, address_book, prefer_immediate);
 

From 996f18995169112c6202dfe1dd474174aab84bdb Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 20 Aug 2022 22:08:26 +0900
Subject: [PATCH 0771/3199] save

---
 scheduler/src/lib.rs | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index e18da2dd90eba3..ee0b2c130750fb 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -210,7 +210,7 @@ type AddressMapEntry<'a> = dashmap::mapref::entry::Entry<'a, Pubkey, PageRc>;
 pub struct AddressBook {
     book: AddressMap,
     uncontended_task_ids: WeightedTaskIds,
-    runnable_provisional_task_ids: WeightedTaskIds,
+    fulfilled_provisional_task_ids: WeightedTaskIds,
     provisioning_trackers: std::collections::HashMap<UniqueWeight, usize>, 
 }
 
@@ -642,8 +642,8 @@ impl ScheduleStage {
         address_book: &mut AddressBook,
         prefer_immediate: bool,
     ) -> Option<(UniqueWeight, Task, Vec<LockAttempt>)> {
-        if let Some(a) = address_book.runnable_provisional_task_ids.pop_last() {
-            trace!("expediate pop from provisional queue [rest: {}]", address_book.runnable_provisional_task_ids.len());
+        if let Some(a) = address_book.fulfilled_provisional_task_ids.pop_last() {
+            trace!("expediate pop from provisional queue [rest: {}]", address_book.fulfilled_provisional_task_ids.len());
             let queue_entry = contended_queue.entry_to_execute(a.0);
             let mut task = queue_entry.remove();
             let ll = std::mem::take(&mut task.tx.1);
@@ -817,13 +817,13 @@ impl ScheduleStage {
                 page.switch_to_next_usage();
                 for task_id in std::mem::take(&mut page.provisional_task_ids).keys() {
                     match address_book.provisioning_trackers.entry(*task_id) {
-                        std::collections::hash_map::Entry::Occupied(mut timer_entry) => {
-                            let count = timer_entry.get_mut();
+                        std::collections::hash_map::Entry::Occupied(mut tracker_entry) => {
+                            let count = tracker_entry.get_mut();
                             *count = count.checked_sub(1).unwrap();
                             if *count == 0 {
                                 trace!("provisional lock decrease: {} => {} (!)", *count + 1, *count);
-                                timer_entry.remove();
-                                address_book.runnable_provisional_task_ids.insert(*task_id, ());
+                                tracker_entry.remove();
+                                address_book.fulfilled_provisional_task_ids.insert(*task_id, ());
                             } else {
                                 trace!("provisional lock decrease: {} => {}", *count + 1, *count);
                             }

From c542d0f05016439b6da3e23f60abf79ae9006594 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 20 Aug 2022 22:47:33 +0900
Subject: [PATCH 0772/3199] save

---
 scheduler/src/lib.rs | 60 ++++++++++++++++++++++----------------------
 1 file changed, 30 insertions(+), 30 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index ee0b2c130750fb..99fc42b0dd74b5 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -176,7 +176,7 @@ pub struct Page {
     current_usage: Usage,
     next_usage: Usage,
     pub contended_unique_weights: TaskIds,
-    provisional_task_ids: WeightedTaskIds,
+    provional_task_ids: WeightedTaskIds,
     //loaded account from Accounts db
     //comulative_cu for qos; i.e. track serialized cumulative keyed by addresses and bail out block
     //producing as soon as any one of cu from the executing thread reaches to the limit
@@ -188,7 +188,7 @@ impl Page {
             current_usage,
             next_usage: Usage::Unused,
             contended_unique_weights: Default::default(),
-            provisional_task_ids: Default::default(),
+            provional_task_ids: Default::default(),
         }
     }
 
@@ -210,8 +210,8 @@ type AddressMapEntry<'a> = dashmap::mapref::entry::Entry<'a, Pubkey, PageRc>;
 pub struct AddressBook {
     book: AddressMap,
     uncontended_task_ids: WeightedTaskIds,
-    fulfilled_provisional_task_ids: WeightedTaskIds,
-    provisioning_trackers: std::collections::HashMap<UniqueWeight, usize>, 
+    runnable_provional_task_ids: WeightedTaskIds,
+    gurantee_timers: std::collections::HashMap<UniqueWeight, usize>, 
 }
 
 impl AddressBook {
@@ -488,7 +488,7 @@ fn attempt_lock_for_execution<'a>(
 ) -> (usize, usize, Vec<LockAttempt>) {
     // no short-cuircuit; we at least all need to add to the contended queue
     let mut unlockable_count = 0;
-    let mut provisional_count = 0;
+    let mut provional_count = 0;
 
     for attempt in placeholder_attempts.iter_mut() {
         AddressBook::attempt_lock_address(from_runnable, prefer_immediate, unique_weight, attempt);
@@ -498,12 +498,12 @@ fn attempt_lock_for_execution<'a>(
                 unlockable_count += 1;
             },
             LockStatus::Guaranteed => {
-                provisional_count += 1;
+                provional_count += 1;
             },
         }
     }
 
-    (unlockable_count, provisional_count, placeholder_attempts)
+    (unlockable_count, provional_count, placeholder_attempts)
 }
 
 type PreprocessedTransaction = (SanitizedTransaction, Vec<LockAttempt>);
@@ -642,8 +642,8 @@ impl ScheduleStage {
         address_book: &mut AddressBook,
         prefer_immediate: bool,
     ) -> Option<(UniqueWeight, Task, Vec<LockAttempt>)> {
-        if let Some(a) = address_book.fulfilled_provisional_task_ids.pop_last() {
-            trace!("expediate pop from provisional queue [rest: {}]", address_book.fulfilled_provisional_task_ids.len());
+        if let Some(a) = address_book.runnable_provional_task_ids.pop_last() {
+            trace!("expediate pop from provisional queue [rest: {}]", address_book.runnable_provional_task_ids.len());
             let queue_entry = contended_queue.entry_to_execute(a.0);
             let mut task = queue_entry.remove();
             let ll = std::mem::take(&mut task.tx.1);
@@ -663,7 +663,7 @@ impl ScheduleStage {
             // plumb message_hash into StatusCache or implmenent our own for duplicate tx
             // detection?
 
-            let (unlockable_count, provisional_count, mut populated_lock_attempts) = attempt_lock_for_execution(
+            let (unlockable_count, provional_count, mut populated_lock_attempts) = attempt_lock_for_execution(
                 from_runnable,
                 prefer_immediate,
                 address_book,
@@ -685,13 +685,13 @@ impl ScheduleStage {
                 next_task.contention_count += 1;
 
                 if from_runnable {
-                    trace!("move to contended due to lock failure [{}/{}/{}]", unlockable_count, provisional_count, lock_count);
+                    trace!("move to contended due to lock failure [{}/{}/{}]", unlockable_count, provional_count, lock_count);
                     reborrowed_contended_queue
                         .unwrap()
                         .add_to_schedule(*queue_entry.key(), queue_entry.remove());
                     // maybe run lightweight prune logic on contended_queue here.
                 } else {
-                    trace!("relock failed [{}/{}/{}]; remains in contended: {:?} contention: {}", unlockable_count, provisional_count, lock_count, &unique_weight, next_task.contention_count);
+                    trace!("relock failed [{}/{}/{}]; remains in contended: {:?} contention: {}", unlockable_count, provional_count, lock_count, &unique_weight, next_task.contention_count);
                     //address_book.uncontended_task_ids.clear();
                 }
 
@@ -700,15 +700,15 @@ impl ScheduleStage {
                 } else {
                     return None;
                 }
-            } else if provisional_count > 0 {
+            } else if provional_count > 0 {
                 assert!(!from_runnable);
                 let lock_count = populated_lock_attempts.len();
-                trace!("provisional exec: [{}/{}]", provisional_count, lock_count);
-                Self::finalize_lock_for_provisional_execution(
+                trace!("provisional exec: [{}/{}]", provional_count, lock_count);
+                Self::finalize_lock_for_provional_execution(
                     address_book,
                     &unique_weight,
                     &mut populated_lock_attempts,
-                    provisional_count,
+                    provional_count,
                 );
                 std::mem::swap(&mut next_task.tx.1, &mut populated_lock_attempts);
 
@@ -746,17 +746,17 @@ impl ScheduleStage {
     }
 
     #[inline(never)]
-    fn finalize_lock_for_provisional_execution(
+    fn finalize_lock_for_provional_execution(
         address_book: &mut AddressBook,
         unique_weight: &UniqueWeight,
         lock_attempts: &mut Vec<LockAttempt>,
-        provisional_count: usize,
+        provional_count: usize,
     ) {
         for mut l in lock_attempts {
             //AddressBook::forget_address_contention(&unique_weight, &mut l);
             match l.status {
                 LockStatus::Guaranteed => {
-                    l.target.page_mut().provisional_task_ids.insert(*unique_weight, ());
+                    l.target.page_mut().provional_task_ids.insert(*unique_weight, ());
                 }
                 LockStatus::Succeded => {
                     // do nothing
@@ -766,8 +766,8 @@ impl ScheduleStage {
                 }
             }
         }
-        address_book.provisioning_trackers.insert(*unique_weight, provisional_count);
-        trace!("provisioning_trackers: {}", address_book.provisioning_trackers.len());
+        address_book.gurantee_timers.insert(*unique_weight, provional_count);
+        trace!("gurantee_timers: {}", address_book.gurantee_timers.len());
     }
 
     #[inline(never)]
@@ -815,15 +815,15 @@ impl ScheduleStage {
             }
             if page.current_usage == Usage::Unused && page.next_usage != Usage::Unused {
                 page.switch_to_next_usage();
-                for task_id in std::mem::take(&mut page.provisional_task_ids).keys() {
-                    match address_book.provisioning_trackers.entry(*task_id) {
-                        std::collections::hash_map::Entry::Occupied(mut tracker_entry) => {
-                            let count = tracker_entry.get_mut();
+                for task_id in std::mem::take(&mut page.provional_task_ids).keys() {
+                    match address_book.gurantee_timers.entry(*task_id) {
+                        std::collections::hash_map::Entry::Occupied(mut timer_entry) => {
+                            let count = timer_entry.get_mut();
                             *count = count.checked_sub(1).unwrap();
                             if *count == 0 {
                                 trace!("provisional lock decrease: {} => {} (!)", *count + 1, *count);
-                                tracker_entry.remove();
-                                address_book.fulfilled_provisional_task_ids.insert(*task_id, ());
+                                timer_entry.remove();
+                                address_book.runnable_provional_task_ids.insert(*task_id, ());
                             } else {
                                 trace!("provisional lock decrease: {} => {}", *count + 1, *count);
                             }
@@ -907,7 +907,7 @@ impl ScheduleStage {
         let mut current_unique_key = u64::max_value();
 
         loop {
-            trace!("schedule_once (from: {}, to: {}, runnnable: {}, contended: {}, (immediate+provisional)/max: ({}+{})/{}) active from contended: {}!", from.len(), to_execute_substage.len(), runnable_queue.task_count(), contended_queue.task_count(), executing_queue_count, address_book.provisioning_trackers.len(), max_executing_queue_count, address_book.uncontended_task_ids.len());
+            trace!("schedule_once (from: {}, to: {}, runnnable: {}, contended: {}, (immediate+provisional)/max: ({}+{})/{}) active from contended: {}!", from.len(), to_execute_substage.len(), runnable_queue.task_count(), contended_queue.task_count(), executing_queue_count, address_book.gurantee_timers.len(), max_executing_queue_count, address_book.uncontended_task_ids.len());
 
             crossbeam_channel::select! {
                recv(from) -> maybe_from => {
@@ -971,7 +971,7 @@ impl ScheduleStage {
             loop {
                 /* if !address_book.uncontended_task_ids.is_empty() {
                     trace!("prefer emptying n_u_a");
-                } else */ if (executing_queue_count + address_book.provisioning_trackers.len()) >= max_executing_queue_count {
+                } else */ if (executing_queue_count + address_book.gurantee_timers.len()) >= max_executing_queue_count {
                     trace!("skip scheduling; outgoing queue full");
                     while from.len() > 0 {
                        let i = from.recv().unwrap();
@@ -988,7 +988,7 @@ impl ScheduleStage {
                     break;
                 }
 
-                let prefer_immediate = address_book.provisioning_trackers.len()/4 > executing_queue_count;
+                let prefer_immediate = address_book.gurantee_timers.len()/4 > executing_queue_count;
                 let maybe_ee =
                     Self::schedule_next_execution(runnable_queue, contended_queue, address_book, prefer_immediate);
 

From 54c48652fd93fd9ad01edc3818b3c82739355332 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 20 Aug 2022 22:48:00 +0900
Subject: [PATCH 0773/3199] save

---
 scheduler/src/lib.rs | 60 ++++++++++++++++++++++----------------------
 1 file changed, 30 insertions(+), 30 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 99fc42b0dd74b5..ee0b2c130750fb 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -176,7 +176,7 @@ pub struct Page {
     current_usage: Usage,
     next_usage: Usage,
     pub contended_unique_weights: TaskIds,
-    provional_task_ids: WeightedTaskIds,
+    provisional_task_ids: WeightedTaskIds,
     //loaded account from Accounts db
     //comulative_cu for qos; i.e. track serialized cumulative keyed by addresses and bail out block
     //producing as soon as any one of cu from the executing thread reaches to the limit
@@ -188,7 +188,7 @@ impl Page {
             current_usage,
             next_usage: Usage::Unused,
             contended_unique_weights: Default::default(),
-            provional_task_ids: Default::default(),
+            provisional_task_ids: Default::default(),
         }
     }
 
@@ -210,8 +210,8 @@ type AddressMapEntry<'a> = dashmap::mapref::entry::Entry<'a, Pubkey, PageRc>;
 pub struct AddressBook {
     book: AddressMap,
     uncontended_task_ids: WeightedTaskIds,
-    runnable_provional_task_ids: WeightedTaskIds,
-    gurantee_timers: std::collections::HashMap<UniqueWeight, usize>, 
+    fulfilled_provisional_task_ids: WeightedTaskIds,
+    provisioning_trackers: std::collections::HashMap<UniqueWeight, usize>, 
 }
 
 impl AddressBook {
@@ -488,7 +488,7 @@ fn attempt_lock_for_execution<'a>(
 ) -> (usize, usize, Vec<LockAttempt>) {
     // no short-cuircuit; we at least all need to add to the contended queue
     let mut unlockable_count = 0;
-    let mut provional_count = 0;
+    let mut provisional_count = 0;
 
     for attempt in placeholder_attempts.iter_mut() {
         AddressBook::attempt_lock_address(from_runnable, prefer_immediate, unique_weight, attempt);
@@ -498,12 +498,12 @@ fn attempt_lock_for_execution<'a>(
                 unlockable_count += 1;
             },
             LockStatus::Guaranteed => {
-                provional_count += 1;
+                provisional_count += 1;
             },
         }
     }
 
-    (unlockable_count, provional_count, placeholder_attempts)
+    (unlockable_count, provisional_count, placeholder_attempts)
 }
 
 type PreprocessedTransaction = (SanitizedTransaction, Vec<LockAttempt>);
@@ -642,8 +642,8 @@ impl ScheduleStage {
         address_book: &mut AddressBook,
         prefer_immediate: bool,
     ) -> Option<(UniqueWeight, Task, Vec<LockAttempt>)> {
-        if let Some(a) = address_book.runnable_provional_task_ids.pop_last() {
-            trace!("expediate pop from provisional queue [rest: {}]", address_book.runnable_provional_task_ids.len());
+        if let Some(a) = address_book.fulfilled_provisional_task_ids.pop_last() {
+            trace!("expediate pop from provisional queue [rest: {}]", address_book.fulfilled_provisional_task_ids.len());
             let queue_entry = contended_queue.entry_to_execute(a.0);
             let mut task = queue_entry.remove();
             let ll = std::mem::take(&mut task.tx.1);
@@ -663,7 +663,7 @@ impl ScheduleStage {
             // plumb message_hash into StatusCache or implmenent our own for duplicate tx
             // detection?
 
-            let (unlockable_count, provional_count, mut populated_lock_attempts) = attempt_lock_for_execution(
+            let (unlockable_count, provisional_count, mut populated_lock_attempts) = attempt_lock_for_execution(
                 from_runnable,
                 prefer_immediate,
                 address_book,
@@ -685,13 +685,13 @@ impl ScheduleStage {
                 next_task.contention_count += 1;
 
                 if from_runnable {
-                    trace!("move to contended due to lock failure [{}/{}/{}]", unlockable_count, provional_count, lock_count);
+                    trace!("move to contended due to lock failure [{}/{}/{}]", unlockable_count, provisional_count, lock_count);
                     reborrowed_contended_queue
                         .unwrap()
                         .add_to_schedule(*queue_entry.key(), queue_entry.remove());
                     // maybe run lightweight prune logic on contended_queue here.
                 } else {
-                    trace!("relock failed [{}/{}/{}]; remains in contended: {:?} contention: {}", unlockable_count, provional_count, lock_count, &unique_weight, next_task.contention_count);
+                    trace!("relock failed [{}/{}/{}]; remains in contended: {:?} contention: {}", unlockable_count, provisional_count, lock_count, &unique_weight, next_task.contention_count);
                     //address_book.uncontended_task_ids.clear();
                 }
 
@@ -700,15 +700,15 @@ impl ScheduleStage {
                 } else {
                     return None;
                 }
-            } else if provional_count > 0 {
+            } else if provisional_count > 0 {
                 assert!(!from_runnable);
                 let lock_count = populated_lock_attempts.len();
-                trace!("provisional exec: [{}/{}]", provional_count, lock_count);
-                Self::finalize_lock_for_provional_execution(
+                trace!("provisional exec: [{}/{}]", provisional_count, lock_count);
+                Self::finalize_lock_for_provisional_execution(
                     address_book,
                     &unique_weight,
                     &mut populated_lock_attempts,
-                    provional_count,
+                    provisional_count,
                 );
                 std::mem::swap(&mut next_task.tx.1, &mut populated_lock_attempts);
 
@@ -746,17 +746,17 @@ impl ScheduleStage {
     }
 
     #[inline(never)]
-    fn finalize_lock_for_provional_execution(
+    fn finalize_lock_for_provisional_execution(
         address_book: &mut AddressBook,
         unique_weight: &UniqueWeight,
         lock_attempts: &mut Vec<LockAttempt>,
-        provional_count: usize,
+        provisional_count: usize,
     ) {
         for mut l in lock_attempts {
             //AddressBook::forget_address_contention(&unique_weight, &mut l);
             match l.status {
                 LockStatus::Guaranteed => {
-                    l.target.page_mut().provional_task_ids.insert(*unique_weight, ());
+                    l.target.page_mut().provisional_task_ids.insert(*unique_weight, ());
                 }
                 LockStatus::Succeded => {
                     // do nothing
@@ -766,8 +766,8 @@ impl ScheduleStage {
                 }
             }
         }
-        address_book.gurantee_timers.insert(*unique_weight, provional_count);
-        trace!("gurantee_timers: {}", address_book.gurantee_timers.len());
+        address_book.provisioning_trackers.insert(*unique_weight, provisional_count);
+        trace!("provisioning_trackers: {}", address_book.provisioning_trackers.len());
     }
 
     #[inline(never)]
@@ -815,15 +815,15 @@ impl ScheduleStage {
             }
             if page.current_usage == Usage::Unused && page.next_usage != Usage::Unused {
                 page.switch_to_next_usage();
-                for task_id in std::mem::take(&mut page.provional_task_ids).keys() {
-                    match address_book.gurantee_timers.entry(*task_id) {
-                        std::collections::hash_map::Entry::Occupied(mut timer_entry) => {
-                            let count = timer_entry.get_mut();
+                for task_id in std::mem::take(&mut page.provisional_task_ids).keys() {
+                    match address_book.provisioning_trackers.entry(*task_id) {
+                        std::collections::hash_map::Entry::Occupied(mut tracker_entry) => {
+                            let count = tracker_entry.get_mut();
                             *count = count.checked_sub(1).unwrap();
                             if *count == 0 {
                                 trace!("provisional lock decrease: {} => {} (!)", *count + 1, *count);
-                                timer_entry.remove();
-                                address_book.runnable_provional_task_ids.insert(*task_id, ());
+                                tracker_entry.remove();
+                                address_book.fulfilled_provisional_task_ids.insert(*task_id, ());
                             } else {
                                 trace!("provisional lock decrease: {} => {}", *count + 1, *count);
                             }
@@ -907,7 +907,7 @@ impl ScheduleStage {
         let mut current_unique_key = u64::max_value();
 
         loop {
-            trace!("schedule_once (from: {}, to: {}, runnnable: {}, contended: {}, (immediate+provisional)/max: ({}+{})/{}) active from contended: {}!", from.len(), to_execute_substage.len(), runnable_queue.task_count(), contended_queue.task_count(), executing_queue_count, address_book.gurantee_timers.len(), max_executing_queue_count, address_book.uncontended_task_ids.len());
+            trace!("schedule_once (from: {}, to: {}, runnnable: {}, contended: {}, (immediate+provisional)/max: ({}+{})/{}) active from contended: {}!", from.len(), to_execute_substage.len(), runnable_queue.task_count(), contended_queue.task_count(), executing_queue_count, address_book.provisioning_trackers.len(), max_executing_queue_count, address_book.uncontended_task_ids.len());
 
             crossbeam_channel::select! {
                recv(from) -> maybe_from => {
@@ -971,7 +971,7 @@ impl ScheduleStage {
             loop {
                 /* if !address_book.uncontended_task_ids.is_empty() {
                     trace!("prefer emptying n_u_a");
-                } else */ if (executing_queue_count + address_book.gurantee_timers.len()) >= max_executing_queue_count {
+                } else */ if (executing_queue_count + address_book.provisioning_trackers.len()) >= max_executing_queue_count {
                     trace!("skip scheduling; outgoing queue full");
                     while from.len() > 0 {
                        let i = from.recv().unwrap();
@@ -988,7 +988,7 @@ impl ScheduleStage {
                     break;
                 }
 
-                let prefer_immediate = address_book.gurantee_timers.len()/4 > executing_queue_count;
+                let prefer_immediate = address_book.provisioning_trackers.len()/4 > executing_queue_count;
                 let maybe_ee =
                     Self::schedule_next_execution(runnable_queue, contended_queue, address_book, prefer_immediate);
 

From 63871ebf04a1b42a833e749f311f430114869cfa Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 21 Aug 2022 22:19:51 +0900
Subject: [PATCH 0774/3199] save

---
 scheduler/src/lib.rs | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index ee0b2c130750fb..2b006c8f363a7b 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -63,7 +63,7 @@ impl PageRc {
 #[derive(Clone, Debug)]
 enum LockStatus {
     Succeded,
-    Guaranteed, 
+    Provisional, 
     Failed,
 }
 
@@ -253,7 +253,7 @@ impl AddressBook {
                             } else {
                                 match page.next_usage {
                                     Usage::Unused => {
-                                        *status = LockStatus::Guaranteed;
+                                        *status = LockStatus::Provisional;
                                         page.next_usage = Usage::renew(*requested_usage);
                                     },
                                     // support multiple readonly locks!
@@ -274,7 +274,7 @@ impl AddressBook {
                         } else {
                             match page.next_usage {
                                 Usage::Unused => {
-                                    *status = LockStatus::Guaranteed;
+                                    *status = LockStatus::Provisional;
                                     page.next_usage = Usage::renew(*requested_usage);
                                 },
                                 // support multiple readonly locks!
@@ -307,7 +307,7 @@ impl AddressBook {
             LockStatus::Succeded => {
                 self.unlock(attempt)
             },
-            LockStatus::Guaranteed => {
+            LockStatus::Provisional => {
                 if after_execution {
                     self.unlock(attempt)
                 } else {
@@ -497,7 +497,7 @@ fn attempt_lock_for_execution<'a>(
             LockStatus::Failed => {
                 unlockable_count += 1;
             },
-            LockStatus::Guaranteed => {
+            LockStatus::Provisional => {
                 provisional_count += 1;
             },
         }
@@ -755,7 +755,7 @@ impl ScheduleStage {
         for mut l in lock_attempts {
             //AddressBook::forget_address_contention(&unique_weight, &mut l);
             match l.status {
-                LockStatus::Guaranteed => {
+                LockStatus::Provisional => {
                     l.target.page_mut().provisional_task_ids.insert(*unique_weight, ());
                 }
                 LockStatus::Succeded => {

From 6462fc9930ffd5f4292d9258c401390a80721ef0 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 21 Aug 2022 22:21:17 +0900
Subject: [PATCH 0775/3199] save

---
 scheduler/src/lib.rs | 1 +
 1 file changed, 1 insertion(+)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 2b006c8f363a7b..f86869ce1d9ea6 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -72,6 +72,7 @@ pub struct LockAttempt {
     pub target: PageRc,
     status: LockStatus,
     requested_usage: RequestedUsage,
+    heaviest_uncontended: AtomicU64,
     //remembered: bool,
 }
 

From 6342548843b764b071f9c79ead49cecb8f76a5c7 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 21 Aug 2022 22:22:07 +0900
Subject: [PATCH 0776/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index f86869ce1d9ea6..58e7cb14a77f98 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -72,7 +72,7 @@ pub struct LockAttempt {
     pub target: PageRc,
     status: LockStatus,
     requested_usage: RequestedUsage,
-    heaviest_uncontended: AtomicU64,
+    heaviest_uncontended: std::sync::atomic::AtomicU64,
     //remembered: bool,
 }
 

From 9e2037d3e58ceaa377fc5e2e9c8c726fbaf585d2 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 21 Aug 2022 22:22:49 +0900
Subject: [PATCH 0777/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 58e7cb14a77f98..e0afae3ce92925 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -72,7 +72,7 @@ pub struct LockAttempt {
     pub target: PageRc,
     status: LockStatus,
     requested_usage: RequestedUsage,
-    heaviest_uncontended: std::sync::atomic::AtomicU64,
+    heaviest_uncontended: std::sync::Arc<std::sync::atomic::AtomicU64>,
     //remembered: bool,
 }
 

From f79d689a499d21227bc63655da4a53b9f9746e7b Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 21 Aug 2022 22:23:17 +0900
Subject: [PATCH 0778/3199] save

---
 scheduler/src/lib.rs | 1 +
 1 file changed, 1 insertion(+)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index e0afae3ce92925..e5a54e917c8dcf 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -82,6 +82,7 @@ impl LockAttempt {
             target,
             status: LockStatus::Succeded,
             requested_usage,
+            heaviest_uncontended: Default::default(),
             //remembered: false,
         }
     }

From dfed3d93f5ef1c99a54c65d4337dea92c237d8f3 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 21 Aug 2022 22:23:36 +0900
Subject: [PATCH 0779/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index e5a54e917c8dcf..10be34c23a35ea 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -224,7 +224,7 @@ impl AddressBook {
         unique_weight: &UniqueWeight,
         attempt: &mut LockAttempt,
     ) {
-        let LockAttempt {target, requested_usage, status/*, remembered*/} = attempt;
+        let LockAttempt {target, requested_usage, status/*, remembered*/, ..} = attempt;
 
                 let mut page = target.page_mut();
 

From 325f0c208faf88887c97675f2d73471a0168a6f7 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 21 Aug 2022 22:25:33 +0900
Subject: [PATCH 0780/3199] save

---
 transaction-scheduler-bench/src/main.rs | 1 +
 1 file changed, 1 insertion(+)

diff --git a/transaction-scheduler-bench/src/main.rs b/transaction-scheduler-bench/src/main.rs
index f29b068a4bdaaa..34fa0bd849f3e6 100644
--- a/transaction-scheduler-bench/src/main.rs
+++ b/transaction-scheduler-bench/src/main.rs
@@ -347,6 +347,7 @@ fn handle_transaction_batch(
     let uq = transaction_batch.unique_weight;
     for lock_attempt in transaction_batch.lock_attempts.iter() {
         lock_attempt.target.page_ref().contended_unique_weights.remove_task_id(&uq);
+        lock_attempt.heaviest_uncontended.store(3);
     }
     completed_transaction_sender.0
         .send(transaction_batch)

From 4b5df782c2d953d33b8c07c7cede68a918db2e22 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 21 Aug 2022 22:25:48 +0900
Subject: [PATCH 0781/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 10be34c23a35ea..ef0798d5f7d6cf 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -72,7 +72,7 @@ pub struct LockAttempt {
     pub target: PageRc,
     status: LockStatus,
     requested_usage: RequestedUsage,
-    heaviest_uncontended: std::sync::Arc<std::sync::atomic::AtomicU64>,
+    pub heaviest_uncontended: std::sync::Arc<std::sync::atomic::AtomicU64>,
     //remembered: bool,
 }
 

From c5ca9d96783b3d1dc06c3253c34bc3aa1a2d758e Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 21 Aug 2022 22:26:26 +0900
Subject: [PATCH 0782/3199] save

---
 transaction-scheduler-bench/src/main.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/transaction-scheduler-bench/src/main.rs b/transaction-scheduler-bench/src/main.rs
index 34fa0bd849f3e6..fb4ba8172d94e7 100644
--- a/transaction-scheduler-bench/src/main.rs
+++ b/transaction-scheduler-bench/src/main.rs
@@ -347,7 +347,7 @@ fn handle_transaction_batch(
     let uq = transaction_batch.unique_weight;
     for lock_attempt in transaction_batch.lock_attempts.iter() {
         lock_attempt.target.page_ref().contended_unique_weights.remove_task_id(&uq);
-        lock_attempt.heaviest_uncontended.store(3);
+        lock_attempt.heaviest_uncontended.store(3, std::sync::atomic::Ordering::Seq);
     }
     completed_transaction_sender.0
         .send(transaction_batch)

From 892a93864effe257532433707ae5f4618d43fda1 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 21 Aug 2022 22:26:34 +0900
Subject: [PATCH 0783/3199] save

---
 transaction-scheduler-bench/src/main.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/transaction-scheduler-bench/src/main.rs b/transaction-scheduler-bench/src/main.rs
index fb4ba8172d94e7..5e4c871cade419 100644
--- a/transaction-scheduler-bench/src/main.rs
+++ b/transaction-scheduler-bench/src/main.rs
@@ -347,7 +347,7 @@ fn handle_transaction_batch(
     let uq = transaction_batch.unique_weight;
     for lock_attempt in transaction_batch.lock_attempts.iter() {
         lock_attempt.target.page_ref().contended_unique_weights.remove_task_id(&uq);
-        lock_attempt.heaviest_uncontended.store(3, std::sync::atomic::Ordering::Seq);
+        lock_attempt.heaviest_uncontended.store(3, std::sync::atomic::Ordering::SeqAcl);
     }
     completed_transaction_sender.0
         .send(transaction_batch)

From a6b9e5175bff02449ed730f090b0aba897908d95 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 21 Aug 2022 22:26:48 +0900
Subject: [PATCH 0784/3199] save

---
 transaction-scheduler-bench/src/main.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/transaction-scheduler-bench/src/main.rs b/transaction-scheduler-bench/src/main.rs
index 5e4c871cade419..01bda9444e8f5e 100644
--- a/transaction-scheduler-bench/src/main.rs
+++ b/transaction-scheduler-bench/src/main.rs
@@ -347,7 +347,7 @@ fn handle_transaction_batch(
     let uq = transaction_batch.unique_weight;
     for lock_attempt in transaction_batch.lock_attempts.iter() {
         lock_attempt.target.page_ref().contended_unique_weights.remove_task_id(&uq);
-        lock_attempt.heaviest_uncontended.store(3, std::sync::atomic::Ordering::SeqAcl);
+        lock_attempt.heaviest_uncontended.store(3, std::sync::atomic::Ordering::SeqCst);
     }
     completed_transaction_sender.0
         .send(transaction_batch)

From 0592b5880c2af60d627de7177dca932e1c556dd4 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 21 Aug 2022 22:27:39 +0900
Subject: [PATCH 0785/3199] save

---
 scheduler/src/lib.rs                    | 14 --------------
 transaction-scheduler-bench/src/main.rs | 14 ++++++++++++++
 2 files changed, 14 insertions(+), 14 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index ef0798d5f7d6cf..f03f0d36b629f5 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -800,20 +800,6 @@ impl ScheduleStage {
 
             let page = l.target.page_mut();
             if newly_uncontended_while_queued && page.next_usage == Usage::Unused {
-                if let Some(mut task_cursor) = page.contended_unique_weights.heaviest_task_cursor() {
-                    let mut found = true;
-                    while !contended_queue.has_task(task_cursor.value()) {
-                        if let Some(new_cursor) = task_cursor.prev() {
-                            task_cursor = new_cursor;
-                        } else {
-                            found = false;
-                            break;
-                        }
-                    }
-                    if found {
-                        address_book.uncontended_task_ids.insert(*(task_cursor.value()), ());
-                    }
-                }
             }
             if page.current_usage == Usage::Unused && page.next_usage != Usage::Unused {
                 page.switch_to_next_usage();
diff --git a/transaction-scheduler-bench/src/main.rs b/transaction-scheduler-bench/src/main.rs
index 01bda9444e8f5e..a34ff604e45bc1 100644
--- a/transaction-scheduler-bench/src/main.rs
+++ b/transaction-scheduler-bench/src/main.rs
@@ -347,6 +347,20 @@ fn handle_transaction_batch(
     let uq = transaction_batch.unique_weight;
     for lock_attempt in transaction_batch.lock_attempts.iter() {
         lock_attempt.target.page_ref().contended_unique_weights.remove_task_id(&uq);
+        if let Some(mut task_cursor) = page.contended_unique_weights.heaviest_task_cursor() {
+            let mut found = true;
+            while !contended_queue.has_task(task_cursor.value()) {
+                if let Some(new_cursor) = task_cursor.prev() {
+                    task_cursor = new_cursor;
+                } else {
+                    found = false;
+                    break;
+                }
+            }
+            if found {
+                address_book.uncontended_task_ids.insert(*(task_cursor.value()), ());
+            }
+        }
         lock_attempt.heaviest_uncontended.store(3, std::sync::atomic::Ordering::SeqCst);
     }
     completed_transaction_sender.0

From be8a3b09f7755b343dd83700578517b68b6a19df Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 21 Aug 2022 22:29:05 +0900
Subject: [PATCH 0786/3199] save

---
 scheduler/src/lib.rs | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index f03f0d36b629f5..f8d7051093d4a0 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -800,6 +800,10 @@ impl ScheduleStage {
 
             let page = l.target.page_mut();
             if newly_uncontended_while_queued && page.next_usage == Usage::Unused {
+                let task_id = l.heaviest_uncontended.load(std::sync::atomic::Ordering::SeqCst);
+                if task_id != 0 {
+                    address_book.uncontended_task_ids.insert(task_id);
+                }
             }
             if page.current_usage == Usage::Unused && page.next_usage != Usage::Unused {
                 page.switch_to_next_usage();

From 4c0142d1b16aeb35bb2ff631749615e29b07437b Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 21 Aug 2022 22:29:12 +0900
Subject: [PATCH 0787/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index f8d7051093d4a0..25b4bd285f5911 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -802,7 +802,7 @@ impl ScheduleStage {
             if newly_uncontended_while_queued && page.next_usage == Usage::Unused {
                 let task_id = l.heaviest_uncontended.load(std::sync::atomic::Ordering::SeqCst);
                 if task_id != 0 {
-                    address_book.uncontended_task_ids.insert(task_id);
+                    address_book.uncontended_task_ids.insert(task_id, ());
                 }
             }
             if page.current_usage == Usage::Unused && page.next_usage != Usage::Unused {

From 1c65ce4f1e7b8b206e47c0bd5604503e5af527ef Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 21 Aug 2022 22:29:56 +0900
Subject: [PATCH 0788/3199] save

---
 transaction-scheduler-bench/src/main.rs | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/transaction-scheduler-bench/src/main.rs b/transaction-scheduler-bench/src/main.rs
index a34ff604e45bc1..02c74019c5b4fe 100644
--- a/transaction-scheduler-bench/src/main.rs
+++ b/transaction-scheduler-bench/src/main.rs
@@ -358,10 +358,9 @@ fn handle_transaction_batch(
                 }
             }
             if found {
-                address_book.uncontended_task_ids.insert(*(task_cursor.value()), ());
+                lock_attempt.heaviest_uncontended.store(*(task_cursor.value()), std::sync::atomic::Ordering::SeqCst);
             }
         }
-        lock_attempt.heaviest_uncontended.store(3, std::sync::atomic::Ordering::SeqCst);
     }
     completed_transaction_sender.0
         .send(transaction_batch)

From 3047455ba960b60106e919f0c5c753c08689a2ef Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 21 Aug 2022 22:30:17 +0900
Subject: [PATCH 0789/3199] save

---
 transaction-scheduler-bench/src/main.rs | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/transaction-scheduler-bench/src/main.rs b/transaction-scheduler-bench/src/main.rs
index 02c74019c5b4fe..8b283691848fc2 100644
--- a/transaction-scheduler-bench/src/main.rs
+++ b/transaction-scheduler-bench/src/main.rs
@@ -346,7 +346,8 @@ fn handle_transaction_batch(
 
     let uq = transaction_batch.unique_weight;
     for lock_attempt in transaction_batch.lock_attempts.iter() {
-        lock_attempt.target.page_ref().contended_unique_weights.remove_task_id(&uq);
+        let page = lock_attempt.target.page_ref();
+        page.contended_unique_weights.remove_task_id(&uq);
         if let Some(mut task_cursor) = page.contended_unique_weights.heaviest_task_cursor() {
             let mut found = true;
             while !contended_queue.has_task(task_cursor.value()) {

From 8fa4c45302d55bdb2f6e73394a5bcdb1a87aa3f2 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 21 Aug 2022 22:30:32 +0900
Subject: [PATCH 0790/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 25b4bd285f5911..3d96f13dc94420 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -166,7 +166,7 @@ impl TaskIds {
     }
 
     #[inline(never)]
-    fn heaviest_task_cursor(&self) -> Option<crossbeam_skiplist::set::Entry<'_, u64>> {
+    pub fn heaviest_task_cursor(&self) -> Option<crossbeam_skiplist::set::Entry<'_, u64>> {
         //self.task_ids.last()
         self.task_ids.back()
         //self.cached_heaviest.as_ref()

From 88ba60f21e68664ba2fe50ba8b059977f3e22740 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 21 Aug 2022 22:34:20 +0900
Subject: [PATCH 0791/3199] save

---
 scheduler/Cargo.toml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/scheduler/Cargo.toml b/scheduler/Cargo.toml
index faf179afdf65d6..9db98da14f8e37 100644
--- a/scheduler/Cargo.toml
+++ b/scheduler/Cargo.toml
@@ -17,6 +17,7 @@ crossbeam-channel = "0.5.5"
 crossbeam-skiplist = { path = "../../../crossbeam/crossbeam-skiplist" }
 dashmap = { version = "4.0.2" }
 rayon = "1.5.3"
+im = { version = "15.1.0", features = ["rayon", "serde"] }
 log = "0.4.17"
 solana-entry = { path = "../entry", version = "=1.11.5" }
 solana-runtime = { path = "../runtime", version = "=1.11.5" }

From 42460fc8b0d61da57a7235af36713cf605cd91a1 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 21 Aug 2022 22:34:42 +0900
Subject: [PATCH 0792/3199] save

---
 Cargo.lock           | 1 +
 scheduler/src/lib.rs | 3 ++-
 2 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/Cargo.lock b/Cargo.lock
index 52d975f3e09388..e959c0290c219e 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -6107,6 +6107,7 @@ dependencies = [
  "crossbeam-channel",
  "crossbeam-skiplist",
  "dashmap",
+ "im",
  "log",
  "rand 0.7.3",
  "rayon",
diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 3d96f13dc94420..aaaa86a89e253e 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -438,7 +438,8 @@ pub struct Task {
 // RunnableQueue, ContendedQueue?
 #[derive(Default)]
 pub struct TaskQueue {
-    tasks: std::collections::BTreeMap<UniqueWeight, Task>,
+    //tasks: std::collections::BTreeMap<UniqueWeight, Task>,
+    tasks: im::OrdMap<UniqueWeight, Task>,
 }
 
 impl TaskQueue {

From 17dd2a0b247327058d3c5ee9c2d4c043050cf821 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 21 Aug 2022 22:36:54 +0900
Subject: [PATCH 0793/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index aaaa86a89e253e..0ba0d4be38f14a 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -439,7 +439,7 @@ pub struct Task {
 #[derive(Default)]
 pub struct TaskQueue {
     //tasks: std::collections::BTreeMap<UniqueWeight, Task>,
-    tasks: im::OrdMap<UniqueWeight, Task>,
+    tasks: im::OrdMap<UniqueWeight, std::sync::Arc<Task>>,
 }
 
 impl TaskQueue {

From ea394c96034f610b6c0e6c46092e8808b2d9148a Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 21 Aug 2022 22:38:01 +0900
Subject: [PATCH 0794/3199] save

---
 scheduler/src/lib.rs | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 0ba0d4be38f14a..69b9cb4fee9f75 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -442,6 +442,8 @@ pub struct TaskQueue {
     tasks: im::OrdMap<UniqueWeight, std::sync::Arc<Task>>,
 }
 
+type TaskQueueEntry = im::ordmap::Entry;
+
 impl TaskQueue {
     #[inline(never)]
     fn add_to_schedule(&mut self, unique_weight: UniqueWeight, task: Task) {
@@ -455,12 +457,10 @@ impl TaskQueue {
         &mut self,
         unique_weight: UniqueWeight,
     ) -> std::collections::btree_map::OccupiedEntry<'_, UniqueWeight, Task> {
-        use std::collections::btree_map::Entry;
-
         let queue_entry = self.tasks.entry(unique_weight);
         match queue_entry {
-            Entry::Occupied(queue_entry) => queue_entry,
-            Entry::Vacant(_queue_entry) => unreachable!(),
+            TaskQueueEntry::Occupied(queue_entry) => queue_entry,
+            TaskQueueEntry::Vacant(_queue_entry) => unreachable!(),
         }
     }
 

From 77926443425a1931b4eeb94a02b17062378899e2 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 21 Aug 2022 22:38:20 +0900
Subject: [PATCH 0795/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 69b9cb4fee9f75..b60e4c15dc871d 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -442,7 +442,7 @@ pub struct TaskQueue {
     tasks: im::OrdMap<UniqueWeight, std::sync::Arc<Task>>,
 }
 
-type TaskQueueEntry = im::ordmap::Entry;
+type TaskQueueEntry<'a> = im::ordmap::Entry<'a>;
 
 impl TaskQueue {
     #[inline(never)]

From 980d49a34b50531d17aa03e81b5f288f75a4c544 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 21 Aug 2022 22:38:34 +0900
Subject: [PATCH 0796/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index b60e4c15dc871d..7728ecfefb5710 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -442,7 +442,7 @@ pub struct TaskQueue {
     tasks: im::OrdMap<UniqueWeight, std::sync::Arc<Task>>,
 }
 
-type TaskQueueEntry<'a> = im::ordmap::Entry<'a>;
+type TaskQueueEntry<'a, K, V> = im::ordmap::Entry<'a, K, V>;
 
 impl TaskQueue {
     #[inline(never)]

From aa6a117ff0a2762cb0222ad87668ed5fe0802dd8 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 21 Aug 2022 22:38:54 +0900
Subject: [PATCH 0797/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 7728ecfefb5710..64c704ec475ef0 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -448,7 +448,7 @@ impl TaskQueue {
     #[inline(never)]
     fn add_to_schedule(&mut self, unique_weight: UniqueWeight, task: Task) {
         //trace!("TaskQueue::add(): {:?}", unique_weight);
-        let pre_existed = self.tasks.insert(unique_weight, task);
+        let pre_existed = self.tasks.insert(unique_weight, Arc::new(task));
         debug_assert!(pre_existed.is_none()); //, "identical shouldn't exist: {:?}", unique_weight);
     }
 

From 4a537317e4b21d6df9801d197e922f015d0366aa Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 21 Aug 2022 22:39:04 +0900
Subject: [PATCH 0798/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 64c704ec475ef0..c6403b38ca7064 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -448,7 +448,7 @@ impl TaskQueue {
     #[inline(never)]
     fn add_to_schedule(&mut self, unique_weight: UniqueWeight, task: Task) {
         //trace!("TaskQueue::add(): {:?}", unique_weight);
-        let pre_existed = self.tasks.insert(unique_weight, Arc::new(task));
+        let pre_existed = self.tasks.insert(unique_weight, std::sync::Arc::new(task));
         debug_assert!(pre_existed.is_none()); //, "identical shouldn't exist: {:?}", unique_weight);
     }
 

From 591f0855943a466d15ef7658060f77ac5cf6e743 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 21 Aug 2022 22:40:13 +0900
Subject: [PATCH 0799/3199] save

---
 scheduler/src/lib.rs | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index c6403b38ca7064..206a405ace2594 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -443,6 +443,7 @@ pub struct TaskQueue {
 }
 
 type TaskQueueEntry<'a, K, V> = im::ordmap::Entry<'a, K, V>;
+type TaskQueueOccupiedEntry<'a, K, V> = im::ordmap::OccupiedEntry<'a, K, V>;
 
 impl TaskQueue {
     #[inline(never)]
@@ -456,7 +457,7 @@ impl TaskQueue {
     fn entry_to_execute(
         &mut self,
         unique_weight: UniqueWeight,
-    ) -> std::collections::btree_map::OccupiedEntry<'_, UniqueWeight, Task> {
+    ) -> TaskQueueOccupiedEntry<'_, UniqueWeight, Task> {
         let queue_entry = self.tasks.entry(unique_weight);
         match queue_entry {
             TaskQueueEntry::Occupied(queue_entry) => queue_entry,

From 3c986f2b11e8343ff32206bf2cbd1f7cadb3ff52 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 21 Aug 2022 22:41:18 +0900
Subject: [PATCH 0800/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 206a405ace2594..4ad3968efd7bb8 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -457,7 +457,7 @@ impl TaskQueue {
     fn entry_to_execute(
         &mut self,
         unique_weight: UniqueWeight,
-    ) -> TaskQueueOccupiedEntry<'_, UniqueWeight, Task> {
+    ) -> TaskQueueOccupiedEntry<'_, UniqueWeight, Arc<Task>> {
         let queue_entry = self.tasks.entry(unique_weight);
         match queue_entry {
             TaskQueueEntry::Occupied(queue_entry) => queue_entry,

From f6199840f43031b531c48a39ecfdb01b0c26b73a Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 21 Aug 2022 22:41:39 +0900
Subject: [PATCH 0801/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 4ad3968efd7bb8..62b1d0124692dc 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -473,7 +473,7 @@ impl TaskQueue {
     fn heaviest_entry_to_execute(
         &mut self,
     ) -> Option<std::collections::btree_map::OccupiedEntry<'_, UniqueWeight, Task>> {
-        self.tasks.last_entry()
+        panic!()//self.tasks.last_entry()
     }
 
     fn task_count(&self) -> usize {

From 2580a17c28b3fe24740114de55cd2e88106f9e2f Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 21 Aug 2022 22:41:50 +0900
Subject: [PATCH 0802/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 62b1d0124692dc..43f2ff88b4b588 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -457,7 +457,7 @@ impl TaskQueue {
     fn entry_to_execute(
         &mut self,
         unique_weight: UniqueWeight,
-    ) -> TaskQueueOccupiedEntry<'_, UniqueWeight, Arc<Task>> {
+    ) -> TaskQueueOccupiedEntry<'_, UniqueWeight, std::sync::Arc<Task>> {
         let queue_entry = self.tasks.entry(unique_weight);
         match queue_entry {
             TaskQueueEntry::Occupied(queue_entry) => queue_entry,

From 30940b5d0533fe777170c7f6664dd3465ae556e3 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 21 Aug 2022 22:42:17 +0900
Subject: [PATCH 0803/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 43f2ff88b4b588..b54835c9f5bcff 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -594,7 +594,7 @@ impl ScheduleStage {
         address_book: &mut AddressBook,
     ) -> Option<(
         Option<&'a mut TaskQueue>,
-        std::collections::btree_map::OccupiedEntry<'a, UniqueWeight, Task>,
+        TaskQueueOccupiedEntry<'a, UniqueWeight, Task>,
     )> {
         match (
             runnable_queue.heaviest_entry_to_execute(),

From 379a706b285c789553f7e08beedd3e908e721a31 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 21 Aug 2022 22:42:34 +0900
Subject: [PATCH 0804/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index b54835c9f5bcff..b76c5a57c71303 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -594,7 +594,7 @@ impl ScheduleStage {
         address_book: &mut AddressBook,
     ) -> Option<(
         Option<&'a mut TaskQueue>,
-        TaskQueueOccupiedEntry<'a, UniqueWeight, Task>,
+        TaskQueueOccupiedEntry<'a, UniqueWeight, std::sync::Arc<Task>>,
     )> {
         match (
             runnable_queue.heaviest_entry_to_execute(),

From bdf5f193be878d27f2cf9aa20b1bfc0d35035bfd Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 21 Aug 2022 22:43:28 +0900
Subject: [PATCH 0805/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index b76c5a57c71303..e4e31481913d65 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -472,7 +472,7 @@ impl TaskQueue {
     #[inline(never)]
     fn heaviest_entry_to_execute(
         &mut self,
-    ) -> Option<std::collections::btree_map::OccupiedEntry<'_, UniqueWeight, Task>> {
+    ) -> Option<TaskQueueOccupiedEntry<'_, UniqueWeight, std::sync::Arc<Task>>> {
         panic!()//self.tasks.last_entry()
     }
 

From 620b1a4ebcfdcfc48c5f2164345fb0eead4b03e0 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 21 Aug 2022 22:44:36 +0900
Subject: [PATCH 0806/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index e4e31481913d65..b28daba6b755cb 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -692,7 +692,7 @@ impl ScheduleStage {
                     trace!("move to contended due to lock failure [{}/{}/{}]", unlockable_count, provisional_count, lock_count);
                     reborrowed_contended_queue
                         .unwrap()
-                        .add_to_schedule(*queue_entry.key(), queue_entry.remove());
+                        .add_to_schedule(*queue_entry.key(), queue_entry.remove().get_mut());
                     // maybe run lightweight prune logic on contended_queue here.
                 } else {
                     trace!("relock failed [{}/{}/{}]; remains in contended: {:?} contention: {}", unlockable_count, provisional_count, lock_count, &unique_weight, next_task.contention_count);

From fe525e50fc49ea53f8f47a23c49a27300ecce19f Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 21 Aug 2022 22:44:55 +0900
Subject: [PATCH 0807/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index b28daba6b755cb..fa218a56d6e64b 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -692,7 +692,7 @@ impl ScheduleStage {
                     trace!("move to contended due to lock failure [{}/{}/{}]", unlockable_count, provisional_count, lock_count);
                     reborrowed_contended_queue
                         .unwrap()
-                        .add_to_schedule(*queue_entry.key(), queue_entry.remove().get_mut());
+                        .add_to_schedule(*queue_entry.key(), std::sync::Arc::get_mut(queue_entry.remove()));
                     // maybe run lightweight prune logic on contended_queue here.
                 } else {
                     trace!("relock failed [{}/{}/{}]; remains in contended: {:?} contention: {}", unlockable_count, provisional_count, lock_count, &unique_weight, next_task.contention_count);

From feff460238aad623680385c292ac609b7e9411b3 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 21 Aug 2022 22:45:07 +0900
Subject: [PATCH 0808/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index fa218a56d6e64b..564a5eb7ef340a 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -692,7 +692,7 @@ impl ScheduleStage {
                     trace!("move to contended due to lock failure [{}/{}/{}]", unlockable_count, provisional_count, lock_count);
                     reborrowed_contended_queue
                         .unwrap()
-                        .add_to_schedule(*queue_entry.key(), std::sync::Arc::get_mut(queue_entry.remove()));
+                        .add_to_schedule(*queue_entry.key(), std::sync::Arc::get_mut(&mut queue_entry.remove()));
                     // maybe run lightweight prune logic on contended_queue here.
                 } else {
                     trace!("relock failed [{}/{}/{}]; remains in contended: {:?} contention: {}", unlockable_count, provisional_count, lock_count, &unique_weight, next_task.contention_count);

From d7e294781b10b3caa9d6815517af239111695dd9 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 21 Aug 2022 22:45:48 +0900
Subject: [PATCH 0809/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 564a5eb7ef340a..30c1e1f44b0f50 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -692,7 +692,7 @@ impl ScheduleStage {
                     trace!("move to contended due to lock failure [{}/{}/{}]", unlockable_count, provisional_count, lock_count);
                     reborrowed_contended_queue
                         .unwrap()
-                        .add_to_schedule(*queue_entry.key(), std::sync::Arc::get_mut(&mut queue_entry.remove()));
+                        .add_to_schedule(*queue_entry.key(), std::sync::Arc::get_mut(&mut queue_entry.remove()).unwrap());
                     // maybe run lightweight prune logic on contended_queue here.
                 } else {
                     trace!("relock failed [{}/{}/{}]; remains in contended: {:?} contention: {}", unlockable_count, provisional_count, lock_count, &unique_weight, next_task.contention_count);

From 5899bcb642f76193d2f2fb7425f08a3ec32c025d Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 21 Aug 2022 22:46:04 +0900
Subject: [PATCH 0810/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 30c1e1f44b0f50..9a9ea0bffcc532 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -692,7 +692,7 @@ impl ScheduleStage {
                     trace!("move to contended due to lock failure [{}/{}/{}]", unlockable_count, provisional_count, lock_count);
                     reborrowed_contended_queue
                         .unwrap()
-                        .add_to_schedule(*queue_entry.key(), std::sync::Arc::get_mut(&mut queue_entry.remove()).unwrap());
+                        .add_to_schedule(*queue_entry.key(), *std::sync::Arc::get_mut(&mut queue_entry.remove()).unwrap());
                     // maybe run lightweight prune logic on contended_queue here.
                 } else {
                     trace!("relock failed [{}/{}/{}]; remains in contended: {:?} contention: {}", unlockable_count, provisional_count, lock_count, &unique_weight, next_task.contention_count);

From 04820d842ba15505c7be6f924e40791238dd64db Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 21 Aug 2022 22:46:54 +0900
Subject: [PATCH 0811/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 9a9ea0bffcc532..f2dfee8bb4e2fe 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -651,7 +651,7 @@ impl ScheduleStage {
             let queue_entry = contended_queue.entry_to_execute(a.0);
             let mut task = queue_entry.remove();
             let ll = std::mem::take(&mut task.tx.1);
-            return Some((a.0, task, ll));
+            return Some((a.0, Arc::get_mut(&mut task), ll));
         }
 
         trace!("pop begin");

From 5c5736fb5acfa62075ae24b84baf2a97cc2c8a8f Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 21 Aug 2022 22:47:16 +0900
Subject: [PATCH 0812/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index f2dfee8bb4e2fe..48ac9cc9974dcc 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -727,7 +727,7 @@ impl ScheduleStage {
                 &mut populated_lock_attempts,
             );
             let task = queue_entry.remove();
-            return Some((unique_weight, task, populated_lock_attempts));
+            return Some((unique_weight, Arc::get_mut(task), populated_lock_attempts));
         } else {
             break;
         }

From 0ab68b2618530cc1045c58201892daadd9a7631e Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 21 Aug 2022 22:47:25 +0900
Subject: [PATCH 0813/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 48ac9cc9974dcc..76e87622d50384 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -727,7 +727,7 @@ impl ScheduleStage {
                 &mut populated_lock_attempts,
             );
             let task = queue_entry.remove();
-            return Some((unique_weight, Arc::get_mut(task), populated_lock_attempts));
+            return Some((unique_weight, std::sync::Arc::get_mut(task), populated_lock_attempts));
         } else {
             break;
         }

From e8fe7957b17f39fea8d03dbacfdf2637be50c587 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 21 Aug 2022 22:47:40 +0900
Subject: [PATCH 0814/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 76e87622d50384..18efcc0416cdca 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -727,7 +727,7 @@ impl ScheduleStage {
                 &mut populated_lock_attempts,
             );
             let task = queue_entry.remove();
-            return Some((unique_weight, std::sync::Arc::get_mut(task), populated_lock_attempts));
+            return Some((unique_weight, std::sync::Arc::get_mut(&mut task).unwrap(), populated_lock_attempts));
         } else {
             break;
         }

From cbd5fbc19c2caa7b233bcd83876a737258a5a96c Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 21 Aug 2022 22:47:49 +0900
Subject: [PATCH 0815/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 18efcc0416cdca..cd9fedbd2875b9 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -727,7 +727,7 @@ impl ScheduleStage {
                 &mut populated_lock_attempts,
             );
             let task = queue_entry.remove();
-            return Some((unique_weight, std::sync::Arc::get_mut(&mut task).unwrap(), populated_lock_attempts));
+            return Some((unique_weight, *std::sync::Arc::get_mut(&mut task).unwrap(), populated_lock_attempts));
         } else {
             break;
         }

From a593ba31583dcc71ee85b75baec851b708935107 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 21 Aug 2022 22:48:00 +0900
Subject: [PATCH 0816/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index cd9fedbd2875b9..6e0b7105f55226 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -651,7 +651,7 @@ impl ScheduleStage {
             let queue_entry = contended_queue.entry_to_execute(a.0);
             let mut task = queue_entry.remove();
             let ll = std::mem::take(&mut task.tx.1);
-            return Some((a.0, Arc::get_mut(&mut task), ll));
+            return Some((a.0, std::sync::Arc::get_mut(&mut task), ll));
         }
 
         trace!("pop begin");

From a3ed471fa153f192bbbc4cef34b9fff2b8273923 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 21 Aug 2022 22:48:09 +0900
Subject: [PATCH 0817/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 6e0b7105f55226..403b16a85e16f2 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -651,7 +651,7 @@ impl ScheduleStage {
             let queue_entry = contended_queue.entry_to_execute(a.0);
             let mut task = queue_entry.remove();
             let ll = std::mem::take(&mut task.tx.1);
-            return Some((a.0, std::sync::Arc::get_mut(&mut task), ll));
+            return Some((a.0, std::sync::Arc::get_mut(&mut task).unwrap(), ll));
         }
 
         trace!("pop begin");

From 874dcb5551e0f10f2e16723bb844965b1032a781 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 21 Aug 2022 22:48:24 +0900
Subject: [PATCH 0818/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 403b16a85e16f2..a64ada5bc49034 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -651,7 +651,7 @@ impl ScheduleStage {
             let queue_entry = contended_queue.entry_to_execute(a.0);
             let mut task = queue_entry.remove();
             let ll = std::mem::take(&mut task.tx.1);
-            return Some((a.0, std::sync::Arc::get_mut(&mut task).unwrap(), ll));
+            return Some((a.0, *std::sync::Arc::get_mut(&mut task).unwrap(), ll));
         }
 
         trace!("pop begin");

From 0f10fc7eaae34fe2fd1e54ec0ca8533f0f9aef97 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 21 Aug 2022 22:49:45 +0900
Subject: [PATCH 0819/3199] save

---
 scheduler/src/lib.rs | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index a64ada5bc49034..82dbed0a6c1ba0 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -650,8 +650,9 @@ impl ScheduleStage {
             trace!("expediate pop from provisional queue [rest: {}]", address_book.fulfilled_provisional_task_ids.len());
             let queue_entry = contended_queue.entry_to_execute(a.0);
             let mut task = queue_entry.remove();
+            let mut task = *std::sync::Arc::get_mut(&mut task).unwrap();
             let ll = std::mem::take(&mut task.tx.1);
-            return Some((a.0, *std::sync::Arc::get_mut(&mut task).unwrap(), ll));
+            return Some((a.0, task, ll));
         }
 
         trace!("pop begin");

From 176c0504d24f59997ec942523c30147932d5f3b4 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 21 Aug 2022 22:50:45 +0900
Subject: [PATCH 0820/3199] save

---
 scheduler/src/lib.rs | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 82dbed0a6c1ba0..cd9fedbd2875b9 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -650,9 +650,8 @@ impl ScheduleStage {
             trace!("expediate pop from provisional queue [rest: {}]", address_book.fulfilled_provisional_task_ids.len());
             let queue_entry = contended_queue.entry_to_execute(a.0);
             let mut task = queue_entry.remove();
-            let mut task = *std::sync::Arc::get_mut(&mut task).unwrap();
             let ll = std::mem::take(&mut task.tx.1);
-            return Some((a.0, task, ll));
+            return Some((a.0, Arc::get_mut(&mut task), ll));
         }
 
         trace!("pop begin");

From c65ac7e3acf0c68018b8a29109da54ffb4aed279 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 21 Aug 2022 22:50:55 +0900
Subject: [PATCH 0821/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index cd9fedbd2875b9..a64ada5bc49034 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -651,7 +651,7 @@ impl ScheduleStage {
             let queue_entry = contended_queue.entry_to_execute(a.0);
             let mut task = queue_entry.remove();
             let ll = std::mem::take(&mut task.tx.1);
-            return Some((a.0, Arc::get_mut(&mut task), ll));
+            return Some((a.0, *std::sync::Arc::get_mut(&mut task).unwrap(), ll));
         }
 
         trace!("pop begin");

From 30b36ae74a71db685984f9d7a400e3dc901b2f85 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 21 Aug 2022 22:52:30 +0900
Subject: [PATCH 0822/3199] save

---
 scheduler/src/lib.rs | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index a64ada5bc49034..02c2ead560fee5 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -439,9 +439,10 @@ pub struct Task {
 #[derive(Default)]
 pub struct TaskQueue {
     //tasks: std::collections::BTreeMap<UniqueWeight, Task>,
-    tasks: im::OrdMap<UniqueWeight, std::sync::Arc<Task>>,
+    tasks: im::OrdMap<UniqueWeight, TaskInQueue,
 }
 
+type TaskInQueue = std::sync::Arc<Task>;
 type TaskQueueEntry<'a, K, V> = im::ordmap::Entry<'a, K, V>;
 type TaskQueueOccupiedEntry<'a, K, V> = im::ordmap::OccupiedEntry<'a, K, V>;
 

From 9841896d5693309bfbb84c6d7cc081e85a74e65b Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 21 Aug 2022 22:52:52 +0900
Subject: [PATCH 0823/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 02c2ead560fee5..9e44171261d966 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -439,7 +439,7 @@ pub struct Task {
 #[derive(Default)]
 pub struct TaskQueue {
     //tasks: std::collections::BTreeMap<UniqueWeight, Task>,
-    tasks: im::OrdMap<UniqueWeight, TaskInQueue,
+    tasks: im::OrdMap<UniqueWeight, TaskInQueue>,
 }
 
 type TaskInQueue = std::sync::Arc<Task>;

From 06783421bf2e1a22523223455af2c4f492928385 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 21 Aug 2022 22:54:53 +0900
Subject: [PATCH 0824/3199] save

---
 scheduler/src/lib.rs | 1 +
 1 file changed, 1 insertion(+)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 9e44171261d966..86d38cd36a5e53 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -662,6 +662,7 @@ impl ScheduleStage {
             let from_runnable = reborrowed_contended_queue.is_some();
             let unique_weight = *queue_entry.key();
             let next_task = queue_entry.get_mut();
+            let next_task = unsafe { TaskInQueue::get_mut_unchecked(&mut next_task) };
             let message_hash = next_task.tx.0.message_hash();
             let placeholder_lock_attempts = std::mem::take(&mut next_task.tx.1);
 

From 1c27f56deb9de25ec0ea6f4e6aa84ee3eb2b421a Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 21 Aug 2022 22:55:25 +0900
Subject: [PATCH 0825/3199] save

---
 scheduler/src/lib.rs | 1 +
 1 file changed, 1 insertion(+)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 86d38cd36a5e53..25c3382a76d70e 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -651,6 +651,7 @@ impl ScheduleStage {
             trace!("expediate pop from provisional queue [rest: {}]", address_book.fulfilled_provisional_task_ids.len());
             let queue_entry = contended_queue.entry_to_execute(a.0);
             let mut task = queue_entry.remove();
+            let mut task = unsafe { TaskInQueue::get_mut_unchecked(&mut task) };
             let ll = std::mem::take(&mut task.tx.1);
             return Some((a.0, *std::sync::Arc::get_mut(&mut task).unwrap(), ll));
         }

From 2821c7ce923c6d2ff06dc2fabbc304964857971c Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 21 Aug 2022 22:55:49 +0900
Subject: [PATCH 0826/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 25c3382a76d70e..9e2a785398a658 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -653,7 +653,7 @@ impl ScheduleStage {
             let mut task = queue_entry.remove();
             let mut task = unsafe { TaskInQueue::get_mut_unchecked(&mut task) };
             let ll = std::mem::take(&mut task.tx.1);
-            return Some((a.0, *std::sync::Arc::get_mut(&mut task).unwrap(), ll));
+            return Some((a.0, task, ll));
         }
 
         trace!("pop begin");

From 9c2497b53343f55ee004c386b5b4dd0cf07c49b7 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 21 Aug 2022 22:56:41 +0900
Subject: [PATCH 0827/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 9e2a785398a658..005386a074c052 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -653,7 +653,7 @@ impl ScheduleStage {
             let mut task = queue_entry.remove();
             let mut task = unsafe { TaskInQueue::get_mut_unchecked(&mut task) };
             let ll = std::mem::take(&mut task.tx.1);
-            return Some((a.0, task, ll));
+            return Some((a.0, *task, ll));
         }
 
         trace!("pop begin");

From 296d4cd4dbf11c428e687082390b4427b960380a Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 22 Aug 2022 08:44:01 +0900
Subject: [PATCH 0828/3199] save

---
 scheduler/src/lib.rs | 9 +++------
 1 file changed, 3 insertions(+), 6 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 005386a074c052..206dce3b6ea321 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -489,7 +489,7 @@ fn attempt_lock_for_execution<'a>(
     address_book: &mut AddressBook,
     unique_weight: &UniqueWeight,
     message_hash: &'a Hash,
-    mut placeholder_attempts: Vec<LockAttempt>,
+    placeholder_attempts: &mut Vec<LockAttempt>,
 ) -> (usize, usize, Vec<LockAttempt>) {
     // no short-cuircuit; we at least all need to add to the contended queue
     let mut unlockable_count = 0;
@@ -651,9 +651,8 @@ impl ScheduleStage {
             trace!("expediate pop from provisional queue [rest: {}]", address_book.fulfilled_provisional_task_ids.len());
             let queue_entry = contended_queue.entry_to_execute(a.0);
             let mut task = queue_entry.remove();
-            let mut task = unsafe { TaskInQueue::get_mut_unchecked(&mut task) };
             let ll = std::mem::take(&mut task.tx.1);
-            return Some((a.0, *task, ll));
+            return Some((a.0, *std::sync::Arc::get_mut(&mut task).unwrap(), ll));
         }
 
         trace!("pop begin");
@@ -663,9 +662,7 @@ impl ScheduleStage {
             let from_runnable = reborrowed_contended_queue.is_some();
             let unique_weight = *queue_entry.key();
             let next_task = queue_entry.get_mut();
-            let next_task = unsafe { TaskInQueue::get_mut_unchecked(&mut next_task) };
             let message_hash = next_task.tx.0.message_hash();
-            let placeholder_lock_attempts = std::mem::take(&mut next_task.tx.1);
 
             // plumb message_hash into StatusCache or implmenent our own for duplicate tx
             // detection?
@@ -676,7 +673,7 @@ impl ScheduleStage {
                 address_book,
                 &unique_weight,
                 &message_hash,
-                placeholder_lock_attempts,
+                &mut next_task.tx.1,
             );
 
             if unlockable_count > 0 {

From 5c8a930c107e79f7b1ac9a02f00a0c0a83f49511 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 22 Aug 2022 08:44:35 +0900
Subject: [PATCH 0829/3199] save

---
 scheduler/src/lib.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 206dce3b6ea321..a872e49651799e 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -490,7 +490,7 @@ fn attempt_lock_for_execution<'a>(
     unique_weight: &UniqueWeight,
     message_hash: &'a Hash,
     placeholder_attempts: &mut Vec<LockAttempt>,
-) -> (usize, usize, Vec<LockAttempt>) {
+) -> (usize, usize) {
     // no short-cuircuit; we at least all need to add to the contended queue
     let mut unlockable_count = 0;
     let mut provisional_count = 0;
@@ -508,7 +508,7 @@ fn attempt_lock_for_execution<'a>(
         }
     }
 
-    (unlockable_count, provisional_count, placeholder_attempts)
+    (unlockable_count, provisional_count)
 }
 
 type PreprocessedTransaction = (SanitizedTransaction, Vec<LockAttempt>);

From bd24d8ea62673561774538c5c0e94084c6f5be79 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 22 Aug 2022 08:46:29 +0900
Subject: [PATCH 0830/3199] save

---
 scheduler/src/lib.rs | 19 ++++++++-----------
 1 file changed, 8 insertions(+), 11 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index a872e49651799e..3cb5f4b514c44d 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -651,8 +651,7 @@ impl ScheduleStage {
             trace!("expediate pop from provisional queue [rest: {}]", address_book.fulfilled_provisional_task_ids.len());
             let queue_entry = contended_queue.entry_to_execute(a.0);
             let mut task = queue_entry.remove();
-            let ll = std::mem::take(&mut task.tx.1);
-            return Some((a.0, *std::sync::Arc::get_mut(&mut task).unwrap(), ll));
+            return Some((a.0, *std::sync::Arc::get_mut(&mut task).unwrap()));
         }
 
         trace!("pop begin");
@@ -667,7 +666,7 @@ impl ScheduleStage {
             // plumb message_hash into StatusCache or implmenent our own for duplicate tx
             // detection?
 
-            let (unlockable_count, provisional_count, mut populated_lock_attempts) = attempt_lock_for_execution(
+            let (unlockable_count, provisional_count) = attempt_lock_for_execution(
                 from_runnable,
                 prefer_immediate,
                 address_book,
@@ -681,11 +680,10 @@ impl ScheduleStage {
                 Self::reset_lock_for_failed_execution(
                     address_book,
                     &unique_weight,
-                    &mut populated_lock_attempts,
+                    &mut next_task.tx.1,
                     from_runnable,
                 );
-                let lock_count = populated_lock_attempts.len();
-                std::mem::swap(&mut next_task.tx.1, &mut populated_lock_attempts);
+                let lock_count = next_task.tx.1.len();
                 next_task.contention_count += 1;
 
                 if from_runnable {
@@ -706,15 +704,14 @@ impl ScheduleStage {
                 }
             } else if provisional_count > 0 {
                 assert!(!from_runnable);
-                let lock_count = populated_lock_attempts.len();
+                let lock_count = next_task.tx.1.len();
                 trace!("provisional exec: [{}/{}]", provisional_count, lock_count);
                 Self::finalize_lock_for_provisional_execution(
                     address_book,
                     &unique_weight,
-                    &mut populated_lock_attempts,
+                    &mut next_task.tx.1,
                     provisional_count,
                 );
-                std::mem::swap(&mut next_task.tx.1, &mut populated_lock_attempts);
 
                 return None;
                 continue;
@@ -724,10 +721,10 @@ impl ScheduleStage {
             Self::finalize_lock_before_execution(
                 address_book,
                 &unique_weight,
-                &mut populated_lock_attempts,
+                &mut next_task.tx.1,
             );
             let task = queue_entry.remove();
-            return Some((unique_weight, *std::sync::Arc::get_mut(&mut task).unwrap(), populated_lock_attempts));
+            return Some((unique_weight, *std::sync::Arc::get_mut(&mut task).unwrap()));
         } else {
             break;
         }

From 9944365ae53c950505a60b4247e27b931410b4ff Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 22 Aug 2022 08:46:44 +0900
Subject: [PATCH 0831/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 3cb5f4b514c44d..9a8dfcde30764e 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -646,7 +646,7 @@ impl ScheduleStage {
         contended_queue: &mut TaskQueue,
         address_book: &mut AddressBook,
         prefer_immediate: bool,
-    ) -> Option<(UniqueWeight, Task, Vec<LockAttempt>)> {
+    ) -> Option<(UniqueWeight, Task)> {
         if let Some(a) = address_book.fulfilled_provisional_task_ids.pop_last() {
             trace!("expediate pop from provisional queue [rest: {}]", address_book.fulfilled_provisional_task_ids.len());
             let queue_entry = contended_queue.entry_to_execute(a.0);

From 2bb59e9c08b573baaa2a9e47445589613de7bb76 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 22 Aug 2022 08:48:27 +0900
Subject: [PATCH 0832/3199] save

---
 scheduler/src/lib.rs | 7 ++-----
 1 file changed, 2 insertions(+), 5 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 9a8dfcde30764e..430efbf1abe4c6 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -835,13 +835,11 @@ impl ScheduleStage {
         address_book: &mut AddressBook,
         unique_weight: UniqueWeight,
         task: Task,
-        lock_attempts: Vec<LockAttempt>,
     ) -> Box<ExecutionEnvironment> {
         let mut rng = rand::thread_rng();
         // load account now from AccountsDb
 
         Box::new(ExecutionEnvironment {
-            lock_attempts,
             task,
             unique_weight,
             cu: rng.gen_range(3, 1000),
@@ -850,9 +848,8 @@ impl ScheduleStage {
 
     #[inline(never)]
     fn commit_result(ee: &mut ExecutionEnvironment, address_book: &mut AddressBook, contended_queue: &TaskQueue) {
-        let lock_attempts = std::mem::take(&mut ee.lock_attempts);
         // do par()-ly?
-        Self::unlock_after_execution(address_book, contended_queue, lock_attempts);
+        Self::unlock_after_execution(address_book, contended_queue, ee.task.lock_attempts);
         // block-wide qos validation will be done here
         // if error risen..:
         //   don't commit the tx for banking and potentially finish scheduling at block max cu
@@ -871,7 +868,7 @@ impl ScheduleStage {
     ) -> Option<Box<ExecutionEnvironment>> {
         let maybe_ee =
             Self::pop_from_queue_then_lock(runnable_queue, contended_queue, address_book, prefer_immediate)
-                .map(|(uw, t, ll)| Self::prepare_scheduled_execution(address_book, uw, t, ll));
+                .map(|(uw, t)| Self::prepare_scheduled_execution(address_book, uw, t));
         maybe_ee
     }
 

From d1c512daa3d661121b553bb069c0f8ad820fddc9 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 22 Aug 2022 08:48:44 +0900
Subject: [PATCH 0833/3199] save

---
 scheduler/src/lib.rs | 1 -
 1 file changed, 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 430efbf1abe4c6..ec00bc432af8d1 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -30,7 +30,6 @@ pub struct PageRc(MyRcInner);
 
 #[derive(Debug)]
 pub struct ExecutionEnvironment {
-    pub lock_attempts: Vec<LockAttempt>,
     //accounts: Vec<i8>,
     pub cu: usize,
     pub unique_weight: UniqueWeight,

From 06cab4bdff38bf356fa1d29973c7fa49b7e718d1 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 22 Aug 2022 08:48:56 +0900
Subject: [PATCH 0834/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index ec00bc432af8d1..d006f4f847ca6a 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -848,7 +848,7 @@ impl ScheduleStage {
     #[inline(never)]
     fn commit_result(ee: &mut ExecutionEnvironment, address_book: &mut AddressBook, contended_queue: &TaskQueue) {
         // do par()-ly?
-        Self::unlock_after_execution(address_book, contended_queue, ee.task.lock_attempts);
+        Self::unlock_after_execution(address_book, contended_queue, ee.task.1.lock_attempts);
         // block-wide qos validation will be done here
         // if error risen..:
         //   don't commit the tx for banking and potentially finish scheduling at block max cu

From a5b466fc9fec1c8e548f20696cb501605825f545 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 22 Aug 2022 08:49:20 +0900
Subject: [PATCH 0835/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index d006f4f847ca6a..ad1e1fd4498998 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -848,7 +848,7 @@ impl ScheduleStage {
     #[inline(never)]
     fn commit_result(ee: &mut ExecutionEnvironment, address_book: &mut AddressBook, contended_queue: &TaskQueue) {
         // do par()-ly?
-        Self::unlock_after_execution(address_book, contended_queue, ee.task.1.lock_attempts);
+        Self::unlock_after_execution(address_book, contended_queue, ee.task.tx.1.lock_attempts);
         // block-wide qos validation will be done here
         // if error risen..:
         //   don't commit the tx for banking and potentially finish scheduling at block max cu

From 14d835da4295ffd3ab89c354bc62f71d9cb4b1f0 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 22 Aug 2022 08:49:34 +0900
Subject: [PATCH 0836/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index ad1e1fd4498998..f317a7c3ade7c6 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -848,7 +848,7 @@ impl ScheduleStage {
     #[inline(never)]
     fn commit_result(ee: &mut ExecutionEnvironment, address_book: &mut AddressBook, contended_queue: &TaskQueue) {
         // do par()-ly?
-        Self::unlock_after_execution(address_book, contended_queue, ee.task.tx.1.lock_attempts);
+        Self::unlock_after_execution(address_book, contended_queue, ee.task.tx.1);
         // block-wide qos validation will be done here
         // if error risen..:
         //   don't commit the tx for banking and potentially finish scheduling at block max cu

From 4fc522cb921f9afb3f64f76bb2fdcf996e0b91cb Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 22 Aug 2022 08:50:29 +0900
Subject: [PATCH 0837/3199] save

---
 scheduler/src/lib.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index f317a7c3ade7c6..593f7bfc0651c1 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -645,12 +645,12 @@ impl ScheduleStage {
         contended_queue: &mut TaskQueue,
         address_book: &mut AddressBook,
         prefer_immediate: bool,
-    ) -> Option<(UniqueWeight, Task)> {
+    ) -> Option<(UniqueWeight, TaskInQueue)> {
         if let Some(a) = address_book.fulfilled_provisional_task_ids.pop_last() {
             trace!("expediate pop from provisional queue [rest: {}]", address_book.fulfilled_provisional_task_ids.len());
             let queue_entry = contended_queue.entry_to_execute(a.0);
             let mut task = queue_entry.remove();
-            return Some((a.0, *std::sync::Arc::get_mut(&mut task).unwrap()));
+            return Some((a.0, task);
         }
 
         trace!("pop begin");

From 51db1def2fc97eea68e52dfb2fef66e09835a423 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 22 Aug 2022 08:50:37 +0900
Subject: [PATCH 0838/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 593f7bfc0651c1..89fa06c66d0e79 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -650,7 +650,7 @@ impl ScheduleStage {
             trace!("expediate pop from provisional queue [rest: {}]", address_book.fulfilled_provisional_task_ids.len());
             let queue_entry = contended_queue.entry_to_execute(a.0);
             let mut task = queue_entry.remove();
-            return Some((a.0, task);
+            return Some((a.0, task));
         }
 
         trace!("pop begin");

From 9a25d1404838451e0a3183e9df29f8ac9eacf902 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 22 Aug 2022 08:50:55 +0900
Subject: [PATCH 0839/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 89fa06c66d0e79..3a9f6acddb597c 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -723,7 +723,7 @@ impl ScheduleStage {
                 &mut next_task.tx.1,
             );
             let task = queue_entry.remove();
-            return Some((unique_weight, *std::sync::Arc::get_mut(&mut task).unwrap()));
+            return Some((unique_weight, task));
         } else {
             break;
         }

From a2d18403c25af273a7c785c0fe883ed2342aaab7 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 22 Aug 2022 08:51:13 +0900
Subject: [PATCH 0840/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 3a9f6acddb597c..90b62e08bfb4dc 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -833,7 +833,7 @@ impl ScheduleStage {
     fn prepare_scheduled_execution(
         address_book: &mut AddressBook,
         unique_weight: UniqueWeight,
-        task: Task,
+        task: TaskInQueue,
     ) -> Box<ExecutionEnvironment> {
         let mut rng = rand::thread_rng();
         // load account now from AccountsDb

From e6e2c61a37efc752c82d116f5d7a5ade9c70d8e1 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 22 Aug 2022 08:51:30 +0900
Subject: [PATCH 0841/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 90b62e08bfb4dc..cbbfb48b53191c 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -33,7 +33,7 @@ pub struct ExecutionEnvironment {
     //accounts: Vec<i8>,
     pub cu: usize,
     pub unique_weight: UniqueWeight,
-    pub task: Task,
+    pub task: TaskInQueue,
 }
 
 impl ExecutionEnvironment {

From e94ba73974bb6bca77c475120c679a615e2dca52 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 22 Aug 2022 08:52:31 +0900
Subject: [PATCH 0842/3199] save

---
 scheduler/src/lib.rs | 1 +
 1 file changed, 1 insertion(+)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index cbbfb48b53191c..9e94c6f091b35b 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -660,6 +660,7 @@ impl ScheduleStage {
             let from_runnable = reborrowed_contended_queue.is_some();
             let unique_weight = *queue_entry.key();
             let next_task = queue_entry.get_mut();
+            let next_task = unsafe { MyRcInner::get_mut_unchecked(&mut next_task) };
             let message_hash = next_task.tx.0.message_hash();
 
             // plumb message_hash into StatusCache or implmenent our own for duplicate tx

From 2fc6775c6dc191ff612174c4385f43673e82a563 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 22 Aug 2022 08:52:45 +0900
Subject: [PATCH 0843/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 9e94c6f091b35b..c7b8a306b72a52 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -660,7 +660,7 @@ impl ScheduleStage {
             let from_runnable = reborrowed_contended_queue.is_some();
             let unique_weight = *queue_entry.key();
             let next_task = queue_entry.get_mut();
-            let next_task = unsafe { MyRcInner::get_mut_unchecked(&mut next_task) };
+            let next_task = unsafe { Arc::get_mut_unchecked(&mut next_task) };
             let message_hash = next_task.tx.0.message_hash();
 
             // plumb message_hash into StatusCache or implmenent our own for duplicate tx

From a605273096689b4e68c5fa4440125f2d15d984f8 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 22 Aug 2022 08:52:54 +0900
Subject: [PATCH 0844/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index c7b8a306b72a52..008896960bc010 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -660,7 +660,7 @@ impl ScheduleStage {
             let from_runnable = reborrowed_contended_queue.is_some();
             let unique_weight = *queue_entry.key();
             let next_task = queue_entry.get_mut();
-            let next_task = unsafe { Arc::get_mut_unchecked(&mut next_task) };
+            let next_task = unsafe { TaskInQueue::get_mut_unchecked(&mut next_task) };
             let message_hash = next_task.tx.0.message_hash();
 
             // plumb message_hash into StatusCache or implmenent our own for duplicate tx

From 082995b62479573d3a71ccfee999ee1d007edf61 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 22 Aug 2022 08:53:31 +0900
Subject: [PATCH 0845/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 008896960bc010..4ddd65302e26d6 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -659,7 +659,7 @@ impl ScheduleStage {
             trace!("pop loop iteration");
             let from_runnable = reborrowed_contended_queue.is_some();
             let unique_weight = *queue_entry.key();
-            let next_task = queue_entry.get_mut();
+            let mut next_task = queue_entry.get_mut();
             let next_task = unsafe { TaskInQueue::get_mut_unchecked(&mut next_task) };
             let message_hash = next_task.tx.0.message_hash();
 

From 59e55c76feac36748073cbb94149f95bfdb48ca2 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 22 Aug 2022 08:54:04 +0900
Subject: [PATCH 0846/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 4ddd65302e26d6..a12f04bf81578e 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -690,7 +690,7 @@ impl ScheduleStage {
                     trace!("move to contended due to lock failure [{}/{}/{}]", unlockable_count, provisional_count, lock_count);
                     reborrowed_contended_queue
                         .unwrap()
-                        .add_to_schedule(*queue_entry.key(), *std::sync::Arc::get_mut(&mut queue_entry.remove()).unwrap());
+                        .add_to_schedule(*queue_entry.key(), queue_entry.remove());
                     // maybe run lightweight prune logic on contended_queue here.
                 } else {
                     trace!("relock failed [{}/{}/{}]; remains in contended: {:?} contention: {}", unlockable_count, provisional_count, lock_count, &unique_weight, next_task.contention_count);

From 8784cd0773a28de91bcd44c9ab79dd7dae131a66 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 22 Aug 2022 08:54:30 +0900
Subject: [PATCH 0847/3199] save

---
 scheduler/src/lib.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index a12f04bf81578e..85b8301b72bec6 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -447,9 +447,9 @@ type TaskQueueOccupiedEntry<'a, K, V> = im::ordmap::OccupiedEntry<'a, K, V>;
 
 impl TaskQueue {
     #[inline(never)]
-    fn add_to_schedule(&mut self, unique_weight: UniqueWeight, task: Task) {
+    fn add_to_schedule(&mut self, unique_weight: UniqueWeight, task: TaskInQueue) {
         //trace!("TaskQueue::add(): {:?}", unique_weight);
-        let pre_existed = self.tasks.insert(unique_weight, std::sync::Arc::new(task));
+        let pre_existed = self.tasks.insert(unique_weight, task);
         debug_assert!(pre_existed.is_none()); //, "identical shouldn't exist: {:?}", unique_weight);
     }
 

From f26611ea01cfa1c7c4aaac9e98053608906cd2b1 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 22 Aug 2022 08:54:51 +0900
Subject: [PATCH 0848/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 85b8301b72bec6..9edd889737d310 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -562,7 +562,7 @@ impl ScheduleStage {
             },
             */
             unique_weight,
-            Task { tx, contention_count: 0 },
+            Arc::new(Task { tx, contention_count: 0 }),
         );
         *unique_key -= 1;
     }

From b74a75654d976644abb38547611c72fcf5fe88c3 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 22 Aug 2022 08:54:59 +0900
Subject: [PATCH 0849/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 9edd889737d310..0c296b5138feac 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -562,7 +562,7 @@ impl ScheduleStage {
             },
             */
             unique_weight,
-            Arc::new(Task { tx, contention_count: 0 }),
+            TaskInQueue::new(Task { tx, contention_count: 0 }),
         );
         *unique_key -= 1;
     }

From 742b04919ac65c2480f901a8d8e0b196eb7a759f Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 22 Aug 2022 08:57:46 +0900
Subject: [PATCH 0850/3199] save

---
 scheduler/src/lib.rs | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 0c296b5138feac..7772ac430ec02d 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -793,7 +793,7 @@ impl ScheduleStage {
     }
 
     #[inline(never)]
-    fn unlock_after_execution(address_book: &mut AddressBook, contended_queue: &TaskQueue, lock_attempts: Vec<LockAttempt>) {
+    fn unlock_after_execution(address_book: &mut AddressBook, contended_queue: &TaskQueue, lock_attempts: &mut Vec<LockAttempt>) {
         for mut l in lock_attempts.into_iter() {
             let newly_uncontended_while_queued = address_book.reset_lock(&mut l, true);
 
@@ -849,7 +849,8 @@ impl ScheduleStage {
     #[inline(never)]
     fn commit_result(ee: &mut ExecutionEnvironment, address_book: &mut AddressBook, contended_queue: &TaskQueue) {
         // do par()-ly?
-        Self::unlock_after_execution(address_book, contended_queue, ee.task.tx.1);
+        let mut task = unsafe { TaskInQueue::get_mut_unchecked(&mut ee.task) };
+        Self::unlock_after_execution(address_book, contended_queue, &mut task.tx.1);
         // block-wide qos validation will be done here
         // if error risen..:
         //   don't commit the tx for banking and potentially finish scheduling at block max cu

From aea4d32f04147090e8a393f13fe72a8c1bb61ed6 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 22 Aug 2022 08:59:09 +0900
Subject: [PATCH 0851/3199] save

---
 transaction-scheduler-bench/src/main.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/transaction-scheduler-bench/src/main.rs b/transaction-scheduler-bench/src/main.rs
index 8b283691848fc2..2bf0ed9c07f197 100644
--- a/transaction-scheduler-bench/src/main.rs
+++ b/transaction-scheduler-bench/src/main.rs
@@ -345,7 +345,7 @@ fn handle_transaction_batch(
         .fetch_add(priority_collected, Ordering::Relaxed);
 
     let uq = transaction_batch.unique_weight;
-    for lock_attempt in transaction_batch.lock_attempts.iter() {
+    for lock_attempt in transaction_batch.task.1.lock_attempts.iter() {
         let page = lock_attempt.target.page_ref();
         page.contended_unique_weights.remove_task_id(&uq);
         if let Some(mut task_cursor) = page.contended_unique_weights.heaviest_task_cursor() {

From 96551e33bd3c2138a6b9acd71331ec0d158ef602 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 22 Aug 2022 08:59:19 +0900
Subject: [PATCH 0852/3199] save

---
 transaction-scheduler-bench/src/main.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/transaction-scheduler-bench/src/main.rs b/transaction-scheduler-bench/src/main.rs
index 2bf0ed9c07f197..b5e92577447ab8 100644
--- a/transaction-scheduler-bench/src/main.rs
+++ b/transaction-scheduler-bench/src/main.rs
@@ -345,7 +345,7 @@ fn handle_transaction_batch(
         .fetch_add(priority_collected, Ordering::Relaxed);
 
     let uq = transaction_batch.unique_weight;
-    for lock_attempt in transaction_batch.task.1.lock_attempts.iter() {
+    for lock_attempt in transaction_batch.task.tx.1.lock_attempts.iter() {
         let page = lock_attempt.target.page_ref();
         page.contended_unique_weights.remove_task_id(&uq);
         if let Some(mut task_cursor) = page.contended_unique_weights.heaviest_task_cursor() {

From c48f3b7634e8f7d40853720e618063a88eb3c309 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 22 Aug 2022 08:59:37 +0900
Subject: [PATCH 0853/3199] save

---
 transaction-scheduler-bench/src/main.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/transaction-scheduler-bench/src/main.rs b/transaction-scheduler-bench/src/main.rs
index b5e92577447ab8..f52cd362a3f0be 100644
--- a/transaction-scheduler-bench/src/main.rs
+++ b/transaction-scheduler-bench/src/main.rs
@@ -345,7 +345,7 @@ fn handle_transaction_batch(
         .fetch_add(priority_collected, Ordering::Relaxed);
 
     let uq = transaction_batch.unique_weight;
-    for lock_attempt in transaction_batch.task.tx.1.lock_attempts.iter() {
+    for lock_attempt in transaction_batch.task.tx.1.iter() {
         let page = lock_attempt.target.page_ref();
         page.contended_unique_weights.remove_task_id(&uq);
         if let Some(mut task_cursor) = page.contended_unique_weights.heaviest_task_cursor() {

From 69839706742328aa956e7583d51b2c24a29d8cb8 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 22 Aug 2022 09:02:08 +0900
Subject: [PATCH 0854/3199] save

---
 scheduler/src/lib.rs | 1 +
 1 file changed, 1 insertion(+)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 7772ac430ec02d..bd48da5ea721c9 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -34,6 +34,7 @@ pub struct ExecutionEnvironment {
     pub cu: usize,
     pub unique_weight: UniqueWeight,
     pub task: TaskInQueue,
+    pub checkpointed_contended_queeu: TaskQueue,
 }
 
 impl ExecutionEnvironment {

From 48a714a265d3d203fc4ca674a8554c8e1c636a01 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 22 Aug 2022 09:03:08 +0900
Subject: [PATCH 0855/3199] save

---
 scheduler/src/lib.rs | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index bd48da5ea721c9..22246fb4ecc6b4 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -34,7 +34,7 @@ pub struct ExecutionEnvironment {
     pub cu: usize,
     pub unique_weight: UniqueWeight,
     pub task: TaskInQueue,
-    pub checkpointed_contended_queeu: TaskQueue,
+    pub checkpointed_contended_queue: TaskQueue,
 }
 
 impl ExecutionEnvironment {
@@ -836,14 +836,17 @@ impl ScheduleStage {
         address_book: &mut AddressBook,
         unique_weight: UniqueWeight,
         task: TaskInQueue,
+        contended_queue: &TaskQueue,
     ) -> Box<ExecutionEnvironment> {
         let mut rng = rand::thread_rng();
         // load account now from AccountsDb
+        let checkpointed_contended_queue = contended_queue.clone();
 
         Box::new(ExecutionEnvironment {
             task,
             unique_weight,
             cu: rng.gen_range(3, 1000),
+            checkpointed_contended_queue,
         })
     }
 
@@ -870,7 +873,7 @@ impl ScheduleStage {
     ) -> Option<Box<ExecutionEnvironment>> {
         let maybe_ee =
             Self::pop_from_queue_then_lock(runnable_queue, contended_queue, address_book, prefer_immediate)
-                .map(|(uw, t)| Self::prepare_scheduled_execution(address_book, uw, t));
+                .map(|(uw, t)| Self::prepare_scheduled_execution(address_book, uw, t, contended_queue));
         maybe_ee
     }
 

From b2cb7ec8fba110a1c2318060f16a59d1450614d0 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 22 Aug 2022 09:03:32 +0900
Subject: [PATCH 0856/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 22246fb4ecc6b4..3a8a562e0dec83 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -840,7 +840,7 @@ impl ScheduleStage {
     ) -> Box<ExecutionEnvironment> {
         let mut rng = rand::thread_rng();
         // load account now from AccountsDb
-        let checkpointed_contended_queue = contended_queue.clone();
+        let checkpointed_contended_queue = (*contended_queue).clone();
 
         Box::new(ExecutionEnvironment {
             task,

From 50f13d790f3df272c994797704e48f7a8a4038cc Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 22 Aug 2022 09:03:50 +0900
Subject: [PATCH 0857/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 3a8a562e0dec83..a07fe5915c20d6 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -436,7 +436,7 @@ pub struct Task {
 }
 
 // RunnableQueue, ContendedQueue?
-#[derive(Default)]
+#[derive(Default, Debug)]
 pub struct TaskQueue {
     //tasks: std::collections::BTreeMap<UniqueWeight, Task>,
     tasks: im::OrdMap<UniqueWeight, TaskInQueue>,

From 638315a0f9fc515dfe0fb3eb778e3383fb3e0d8b Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 22 Aug 2022 09:04:04 +0900
Subject: [PATCH 0858/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index a07fe5915c20d6..27b4f6b5469064 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -436,7 +436,7 @@ pub struct Task {
 }
 
 // RunnableQueue, ContendedQueue?
-#[derive(Default, Debug)]
+#[derive(Default, Debug, Clone)]
 pub struct TaskQueue {
     //tasks: std::collections::BTreeMap<UniqueWeight, Task>,
     tasks: im::OrdMap<UniqueWeight, TaskInQueue>,

From 126855e9f95463fea44344aca848fd76deffc421 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 22 Aug 2022 09:04:28 +0900
Subject: [PATCH 0859/3199] save

---
 transaction-scheduler-bench/src/main.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/transaction-scheduler-bench/src/main.rs b/transaction-scheduler-bench/src/main.rs
index f52cd362a3f0be..8cd003d5ccfd0e 100644
--- a/transaction-scheduler-bench/src/main.rs
+++ b/transaction-scheduler-bench/src/main.rs
@@ -350,7 +350,7 @@ fn handle_transaction_batch(
         page.contended_unique_weights.remove_task_id(&uq);
         if let Some(mut task_cursor) = page.contended_unique_weights.heaviest_task_cursor() {
             let mut found = true;
-            while !contended_queue.has_task(task_cursor.value()) {
+            while !transaction_batch.checkpointed_contended_queue.has_task(task_cursor.value()) {
                 if let Some(new_cursor) = task_cursor.prev() {
                     task_cursor = new_cursor;
                 } else {

From 09adb4d76be22be58e80757ed0e52ddfd9438c24 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 22 Aug 2022 09:04:47 +0900
Subject: [PATCH 0860/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 27b4f6b5469064..4852a90e864e6e 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -466,7 +466,7 @@ impl TaskQueue {
         }
     }
 
-    fn has_task(&self, unique_weight: &UniqueWeight) -> bool {
+    pub fn has_task(&self, unique_weight: &UniqueWeight) -> bool {
         self.tasks.contains_key(unique_weight)
     }
 

From 5df4e7d261c8ceffcd43e46bbdbd59a46b91177f Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 22 Aug 2022 09:05:30 +0900
Subject: [PATCH 0861/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 4852a90e864e6e..9e68bc3e1751da 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -474,7 +474,7 @@ impl TaskQueue {
     fn heaviest_entry_to_execute(
         &mut self,
     ) -> Option<TaskQueueOccupiedEntry<'_, UniqueWeight, std::sync::Arc<Task>>> {
-        panic!()//self.tasks.last_entry()
+        self.tasks.last_entry()
     }
 
     fn task_count(&self) -> usize {

From 5d0354a13fb4360cc36c1530a38e9ec83fa464ab Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 22 Aug 2022 09:06:32 +0900
Subject: [PATCH 0862/3199] save

---
 scheduler/src/lib.rs | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 9e68bc3e1751da..1220aab8141e5c 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -458,7 +458,7 @@ impl TaskQueue {
     fn entry_to_execute(
         &mut self,
         unique_weight: UniqueWeight,
-    ) -> TaskQueueOccupiedEntry<'_, UniqueWeight, std::sync::Arc<Task>> {
+    ) -> TaskQueueOccupiedEntry<'_, UniqueWeight, TaskInQueue> {
         let queue_entry = self.tasks.entry(unique_weight);
         match queue_entry {
             TaskQueueEntry::Occupied(queue_entry) => queue_entry,
@@ -473,8 +473,8 @@ impl TaskQueue {
     #[inline(never)]
     fn heaviest_entry_to_execute(
         &mut self,
-    ) -> Option<TaskQueueOccupiedEntry<'_, UniqueWeight, std::sync::Arc<Task>>> {
-        self.tasks.last_entry()
+    ) -> Option<TaskQueueOccupiedEntry<'_, UniqueWeight, TaskInQueue>> {
+        panic!()//self.tasks.last_entry()
     }
 
     fn task_count(&self) -> usize {
@@ -595,7 +595,7 @@ impl ScheduleStage {
         address_book: &mut AddressBook,
     ) -> Option<(
         Option<&'a mut TaskQueue>,
-        TaskQueueOccupiedEntry<'a, UniqueWeight, std::sync::Arc<Task>>,
+        TaskQueueOccupiedEntry<'a, UniqueWeight, TaskInQueue>,
     )> {
         match (
             runnable_queue.heaviest_entry_to_execute(),

From 284f1a242a6242877d1fd32115ba27fa22386505 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 22 Aug 2022 09:08:55 +0900
Subject: [PATCH 0863/3199] save

---
 scheduler/src/lib.rs | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 1220aab8141e5c..0b25dc423bb77a 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -474,7 +474,8 @@ impl TaskQueue {
     fn heaviest_entry_to_execute(
         &mut self,
     ) -> Option<TaskQueueOccupiedEntry<'_, UniqueWeight, TaskInQueue>> {
-        panic!()//self.tasks.last_entry()
+        //panic!()//self.tasks.last_entry()
+        self.tasks.get_max().map(|v| self.entry_to_execute(v))
     }
 
     fn task_count(&self) -> usize {

From f7cbcc3e9715acdf37444daea372110947924423 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 22 Aug 2022 09:09:11 +0900
Subject: [PATCH 0864/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 0b25dc423bb77a..d66a88f65e9d06 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -475,7 +475,7 @@ impl TaskQueue {
         &mut self,
     ) -> Option<TaskQueueOccupiedEntry<'_, UniqueWeight, TaskInQueue>> {
         //panic!()//self.tasks.last_entry()
-        self.tasks.get_max().map(|v| self.entry_to_execute(v))
+        self.tasks.get_max().map(|(k, _v)| self.entry_to_execute(v))
     }
 
     fn task_count(&self) -> usize {

From 2edfa3c22f22ad3e0aa37b79060e61b060e1163d Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 22 Aug 2022 09:09:20 +0900
Subject: [PATCH 0865/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index d66a88f65e9d06..ad0aeab0e13e8b 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -475,7 +475,7 @@ impl TaskQueue {
         &mut self,
     ) -> Option<TaskQueueOccupiedEntry<'_, UniqueWeight, TaskInQueue>> {
         //panic!()//self.tasks.last_entry()
-        self.tasks.get_max().map(|(k, _v)| self.entry_to_execute(v))
+        self.tasks.get_max().map(|(k, _v)| self.entry_to_execute(k))
     }
 
     fn task_count(&self) -> usize {

From 7740de5185ae865500fced99a112e42934e06fc2 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 22 Aug 2022 09:09:28 +0900
Subject: [PATCH 0866/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index ad0aeab0e13e8b..ee512db5d537d4 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -475,7 +475,7 @@ impl TaskQueue {
         &mut self,
     ) -> Option<TaskQueueOccupiedEntry<'_, UniqueWeight, TaskInQueue>> {
         //panic!()//self.tasks.last_entry()
-        self.tasks.get_max().map(|(k, _v)| self.entry_to_execute(k))
+        self.tasks.get_max().map(|(k, _v)| self.entry_to_execute(*k))
     }
 
     fn task_count(&self) -> usize {

From 2ab6382b49a8ae4515d8483db23b4ca9e1d14779 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 22 Aug 2022 09:10:08 +0900
Subject: [PATCH 0867/3199] save

---
 scheduler/src/lib.rs | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index ee512db5d537d4..40daf2d4832b20 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -475,7 +475,8 @@ impl TaskQueue {
         &mut self,
     ) -> Option<TaskQueueOccupiedEntry<'_, UniqueWeight, TaskInQueue>> {
         //panic!()//self.tasks.last_entry()
-        self.tasks.get_max().map(|(k, _v)| self.entry_to_execute(*k))
+        let k = self.tasks.get_max();
+        k.map(|(k, _v)| self.entry_to_execute(*k))
     }
 
     fn task_count(&self) -> usize {

From 46bf91b732ec9bbe8ff2c8e9d7b189e0fcde9bf2 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 22 Aug 2022 09:10:28 +0900
Subject: [PATCH 0868/3199] save

---
 scheduler/src/lib.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 40daf2d4832b20..b0a5c78ea7e448 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -475,8 +475,8 @@ impl TaskQueue {
         &mut self,
     ) -> Option<TaskQueueOccupiedEntry<'_, UniqueWeight, TaskInQueue>> {
         //panic!()//self.tasks.last_entry()
-        let k = self.tasks.get_max();
-        k.map(|(k, _v)| self.entry_to_execute(*k))
+        let k = self.tasks.get_max().map(|(k, _v)| *k);
+        self.entry_to_execute(k);
     }
 
     fn task_count(&self) -> usize {

From 372f595efc7523ab2a4e70e4c0a37517e376bad5 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 22 Aug 2022 09:10:35 +0900
Subject: [PATCH 0869/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index b0a5c78ea7e448..05ab60f8246a3c 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -476,7 +476,7 @@ impl TaskQueue {
     ) -> Option<TaskQueueOccupiedEntry<'_, UniqueWeight, TaskInQueue>> {
         //panic!()//self.tasks.last_entry()
         let k = self.tasks.get_max().map(|(k, _v)| *k);
-        self.entry_to_execute(k);
+        self.entry_to_execute(k)
     }
 
     fn task_count(&self) -> usize {

From 4ef0edaa9bf00227cc36ee6c577907c96cff4f7d Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 22 Aug 2022 09:10:49 +0900
Subject: [PATCH 0870/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 05ab60f8246a3c..9add6f7954791a 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -476,7 +476,7 @@ impl TaskQueue {
     ) -> Option<TaskQueueOccupiedEntry<'_, UniqueWeight, TaskInQueue>> {
         //panic!()//self.tasks.last_entry()
         let k = self.tasks.get_max().map(|(k, _v)| *k);
-        self.entry_to_execute(k)
+        k.map(|k| self.entry_to_execute(k))
     }
 
     fn task_count(&self) -> usize {

From db6de9e8cc7517950355aa7b1f2afe40dd1c8b93 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 22 Aug 2022 12:31:01 +0900
Subject: [PATCH 0871/3199] save

---
 scheduler/src/lib.rs | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 9add6f7954791a..c34ca96715c592 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -920,6 +920,8 @@ impl ScheduleStage {
                                 // async-ly propagate the result to rpc subsystems
                                 if let Some(to_next_stage) = to_next_stage {
                                     to_next_stage.send(processed_execution_environment).unwrap();
+                                } else {
+                                    panic!();
                                 }
                             }
 
@@ -953,6 +955,8 @@ impl ScheduleStage {
                         // async-ly propagate the result to rpc subsystems
                         if let Some(to_next_stage) = to_next_stage {
                             to_next_stage.send(processed_execution_environment).unwrap();
+                        } else {
+                            panic!();
                         }
                         if false && from_exec.len() > 0 {
                             processed_execution_environment = from_exec.recv().unwrap();

From be0a1eba971fc80bdf125377a128b0090ade96e0 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 22 Aug 2022 12:33:01 +0900
Subject: [PATCH 0872/3199] save

---
 scheduler/src/lib.rs | 1 +
 1 file changed, 1 insertion(+)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index c34ca96715c592..3f6f58a8effe67 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -900,6 +900,7 @@ impl ScheduleStage {
     ) {
         let mut executing_queue_count = 0;
         let mut current_unique_key = u64::max_value();
+        let background_ee_reaper = to_next_stage.is_some().then_some(|| 3);
 
         loop {
             trace!("schedule_once (from: {}, to: {}, runnnable: {}, contended: {}, (immediate+provisional)/max: ({}+{})/{}) active from contended: {}!", from.len(), to_execute_substage.len(), runnable_queue.task_count(), contended_queue.task_count(), executing_queue_count, address_book.provisioning_trackers.len(), max_executing_queue_count, address_book.uncontended_task_ids.len());

From f03e7af9bb7032d69867b32e947c40c917d4ce56 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 22 Aug 2022 12:34:25 +0900
Subject: [PATCH 0873/3199] save

---
 scheduler/src/lib.rs | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 3f6f58a8effe67..bfe736eb4b49f5 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -900,7 +900,12 @@ impl ScheduleStage {
     ) {
         let mut executing_queue_count = 0;
         let mut current_unique_key = u64::max_value();
-        let background_ee_reaper = to_next_stage.is_some().then_some(|| 3);
+        let background_ee_reaper = to_next_stage.is_none().then_some(|| {
+            let h = std::thread::Builder::new().name("sol-reaper".to_string()).spawn(move || {
+            }).unwrap();
+
+            (h)
+        });
 
         loop {
             trace!("schedule_once (from: {}, to: {}, runnnable: {}, contended: {}, (immediate+provisional)/max: ({}+{})/{}) active from contended: {}!", from.len(), to_execute_substage.len(), runnable_queue.task_count(), contended_queue.task_count(), executing_queue_count, address_book.provisioning_trackers.len(), max_executing_queue_count, address_book.uncontended_task_ids.len());

From b56e2e8db1a055d5007cfd176a3eea7f3266c135 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 22 Aug 2022 12:35:35 +0900
Subject: [PATCH 0874/3199] save

---
 scheduler/src/lib.rs | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index bfe736eb4b49f5..6ab3a00d8191e9 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -902,6 +902,8 @@ impl ScheduleStage {
         let mut current_unique_key = u64::max_value();
         let background_ee_reaper = to_next_stage.is_none().then_some(|| {
             let h = std::thread::Builder::new().name("sol-reaper".to_string()).spawn(move || {
+                while a = to_reaper.recv().unwrap() {
+                }
             }).unwrap();
 
             (h)

From c662e0ff968d93d5789b21b6488c3bfa90a5a9de Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 22 Aug 2022 12:35:46 +0900
Subject: [PATCH 0875/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 6ab3a00d8191e9..9ea625e99d1f85 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -902,7 +902,7 @@ impl ScheduleStage {
         let mut current_unique_key = u64::max_value();
         let background_ee_reaper = to_next_stage.is_none().then_some(|| {
             let h = std::thread::Builder::new().name("sol-reaper".to_string()).spawn(move || {
-                while a = to_reaper.recv().unwrap() {
+                while let a = to_reaper.recv().unwrap() {
                 }
             }).unwrap();
 

From 575802d31ba4a432ba1693592dbaa87e523ea6c9 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 22 Aug 2022 12:37:00 +0900
Subject: [PATCH 0876/3199] save

---
 scheduler/src/lib.rs | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 9ea625e99d1f85..b1f68c81b1bfde 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -901,12 +901,13 @@ impl ScheduleStage {
         let mut executing_queue_count = 0;
         let mut current_unique_key = u64::max_value();
         let background_ee_reaper = to_next_stage.is_none().then_some(|| {
+            let (ee_sender, ee_receiver) = crossbeam_channel::unbound();
             let h = std::thread::Builder::new().name("sol-reaper".to_string()).spawn(move || {
-                while let a = to_reaper.recv().unwrap() {
+                while let a = ee_receiver.recv().unwrap() {
                 }
             }).unwrap();
 
-            (h)
+            (h, ee_sender)
         });
 
         loop {

From 8c1fb5eac19d6cd9af2c4747ef26d5d0419fd549 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 22 Aug 2022 12:37:10 +0900
Subject: [PATCH 0877/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index b1f68c81b1bfde..10ed1436a72442 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -901,7 +901,7 @@ impl ScheduleStage {
         let mut executing_queue_count = 0;
         let mut current_unique_key = u64::max_value();
         let background_ee_reaper = to_next_stage.is_none().then_some(|| {
-            let (ee_sender, ee_receiver) = crossbeam_channel::unbound();
+            let (ee_sender, ee_receiver) = crossbeam_channel::unbounded();
             let h = std::thread::Builder::new().name("sol-reaper".to_string()).spawn(move || {
                 while let a = ee_receiver.recv().unwrap() {
                 }

From fbf0c0a4608ae391ec860484ff3a28f8f8e21135 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 22 Aug 2022 12:37:46 +0900
Subject: [PATCH 0878/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 10ed1436a72442..e6fb43223d754c 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -901,7 +901,7 @@ impl ScheduleStage {
         let mut executing_queue_count = 0;
         let mut current_unique_key = u64::max_value();
         let background_ee_reaper = to_next_stage.is_none().then_some(|| {
-            let (ee_sender, ee_receiver) = crossbeam_channel::unbounded();
+            let (ee_sender, ee_receiver) = crossbeam_channel::unbounded::<Box<ExecutionEnvironment>>();
             let h = std::thread::Builder::new().name("sol-reaper".to_string()).spawn(move || {
                 while let a = ee_receiver.recv().unwrap() {
                 }

From 3bf5fdab972951d5fb0734292cab8c0579797d50 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 22 Aug 2022 12:38:03 +0900
Subject: [PATCH 0879/3199] save

---
 scheduler/src/lib.rs | 1 +
 1 file changed, 1 insertion(+)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index e6fb43223d754c..d927b02493381b 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -904,6 +904,7 @@ impl ScheduleStage {
             let (ee_sender, ee_receiver) = crossbeam_channel::unbounded::<Box<ExecutionEnvironment>>();
             let h = std::thread::Builder::new().name("sol-reaper".to_string()).spawn(move || {
                 while let a = ee_receiver.recv().unwrap() {
+                    drop(a);
                 }
             }).unwrap();
 

From bfd9933a2e04ac7358306faa78f8c7ceafb5bcc4 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 22 Aug 2022 12:39:00 +0900
Subject: [PATCH 0880/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index d927b02493381b..1b7e350d58ff33 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -966,7 +966,7 @@ impl ScheduleStage {
                         if let Some(to_next_stage) = to_next_stage {
                             to_next_stage.send(processed_execution_environment).unwrap();
                         } else {
-                            panic!();
+                            to_background_reaper.send(processed_execution_environment).unwrap();
                         }
                         if false && from_exec.len() > 0 {
                             processed_execution_environment = from_exec.recv().unwrap();

From 803acdbdbf1f5c906c60d5a99de2bff95283f8cc Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 22 Aug 2022 12:41:25 +0900
Subject: [PATCH 0881/3199] save

---
 scheduler/src/lib.rs | 22 ++++++++++------------
 1 file changed, 10 insertions(+), 12 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 1b7e350d58ff33..17a83838c715c1 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -896,11 +896,11 @@ impl ScheduleStage {
         from: &crossbeam_channel::Receiver<Multiplexed>,
         from_exec: &crossbeam_channel::Receiver<Box<ExecutionEnvironment>>,
         to_execute_substage: &crossbeam_channel::Sender<Box<ExecutionEnvironment>>,
-        to_next_stage: Option<&crossbeam_channel::Sender<Box<ExecutionEnvironment>>>, // assume nonblocking
+        maybe_to_next_stage: Option<&crossbeam_channel::Sender<Box<ExecutionEnvironment>>>, // assume nonblocking
     ) {
         let mut executing_queue_count = 0;
         let mut current_unique_key = u64::max_value();
-        let background_ee_reaper = to_next_stage.is_none().then_some(|| {
+        let background_ee_reaper = maybe_to_next_stage.is_none().then_some(|| {
             let (ee_sender, ee_receiver) = crossbeam_channel::unbounded::<Box<ExecutionEnvironment>>();
             let h = std::thread::Builder::new().name("sol-reaper".to_string()).spawn(move || {
                 while let a = ee_receiver.recv().unwrap() {
@@ -911,6 +911,12 @@ impl ScheduleStage {
             (h, ee_sender)
         });
 
+        let to_next_stage = if let Some(to_next_stage) = maybe_to_next_stage {
+            to_next_stage
+        } else {
+            background_ee_reaper.unwrap().1
+        }
+
         loop {
             trace!("schedule_once (from: {}, to: {}, runnnable: {}, contended: {}, (immediate+provisional)/max: ({}+{})/{}) active from contended: {}!", from.len(), to_execute_substage.len(), runnable_queue.task_count(), contended_queue.task_count(), executing_queue_count, address_book.provisioning_trackers.len(), max_executing_queue_count, address_book.uncontended_task_ids.len());
 
@@ -928,11 +934,7 @@ impl ScheduleStage {
 
                                 Self::commit_result(&mut processed_execution_environment, address_book, contended_queue);
                                 // async-ly propagate the result to rpc subsystems
-                                if let Some(to_next_stage) = to_next_stage {
-                                    to_next_stage.send(processed_execution_environment).unwrap();
-                                } else {
-                                    panic!();
-                                }
+                                to_next_stage.send(processed_execution_environment).unwrap();
                             }
 
                             Self::register_runnable_task(weighted_tx, runnable_queue, &mut current_unique_key);
@@ -963,11 +965,7 @@ impl ScheduleStage {
 
                         Self::commit_result(&mut processed_execution_environment, address_book, contended_queue);
                         // async-ly propagate the result to rpc subsystems
-                        if let Some(to_next_stage) = to_next_stage {
-                            to_next_stage.send(processed_execution_environment).unwrap();
-                        } else {
-                            to_background_reaper.send(processed_execution_environment).unwrap();
-                        }
+                        to_next_stage.send(processed_execution_environment).unwrap();
                         if false && from_exec.len() > 0 {
                             processed_execution_environment = from_exec.recv().unwrap();
                         } else {

From acb6af3fc4f517e6dd0a343835c7afdea6151729 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 22 Aug 2022 12:41:47 +0900
Subject: [PATCH 0882/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 17a83838c715c1..45ec4d8af1d758 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -900,7 +900,7 @@ impl ScheduleStage {
     ) {
         let mut executing_queue_count = 0;
         let mut current_unique_key = u64::max_value();
-        let background_ee_reaper = maybe_to_next_stage.is_none().then_some(|| {
+        let background_ee_reaper = maybe_to_next_stage.is_none().then_else(|| {
             let (ee_sender, ee_receiver) = crossbeam_channel::unbounded::<Box<ExecutionEnvironment>>();
             let h = std::thread::Builder::new().name("sol-reaper".to_string()).spawn(move || {
                 while let a = ee_receiver.recv().unwrap() {

From 806613e695a992f8cd4b71390be87cde7af77154 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 22 Aug 2022 12:42:05 +0900
Subject: [PATCH 0883/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 45ec4d8af1d758..2626688e5f4016 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -900,7 +900,7 @@ impl ScheduleStage {
     ) {
         let mut executing_queue_count = 0;
         let mut current_unique_key = u64::max_value();
-        let background_ee_reaper = maybe_to_next_stage.is_none().then_else(|| {
+        let background_ee_reaper = maybe_to_next_stage.is_none().then(|| {
             let (ee_sender, ee_receiver) = crossbeam_channel::unbounded::<Box<ExecutionEnvironment>>();
             let h = std::thread::Builder::new().name("sol-reaper".to_string()).spawn(move || {
                 while let a = ee_receiver.recv().unwrap() {

From f47e8753aca2456b46791a8cbfd40d053678f7b9 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 22 Aug 2022 12:42:21 +0900
Subject: [PATCH 0884/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 2626688e5f4016..836c3f12831ebf 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -914,7 +914,7 @@ impl ScheduleStage {
         let to_next_stage = if let Some(to_next_stage) = maybe_to_next_stage {
             to_next_stage
         } else {
-            background_ee_reaper.unwrap().1
+            &background_ee_reaper.unwrap().1
         }
 
         loop {

From d512b53481c7397b582d1da55d6ee6917acf0528 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 22 Aug 2022 12:42:28 +0900
Subject: [PATCH 0885/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 836c3f12831ebf..8fb8abe7dcb877 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -915,7 +915,7 @@ impl ScheduleStage {
             to_next_stage
         } else {
             &background_ee_reaper.unwrap().1
-        }
+        };
 
         loop {
             trace!("schedule_once (from: {}, to: {}, runnnable: {}, contended: {}, (immediate+provisional)/max: ({}+{})/{}) active from contended: {}!", from.len(), to_execute_substage.len(), runnable_queue.task_count(), contended_queue.task_count(), executing_queue_count, address_book.provisioning_trackers.len(), max_executing_queue_count, address_book.uncontended_task_ids.len());

From a56942a01b7d8892fc63ebfd8917354960242e0e Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 22 Aug 2022 12:42:39 +0900
Subject: [PATCH 0886/3199] save

---
 scheduler/src/lib.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 8fb8abe7dcb877..3f82d8bbb6c2fb 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -912,9 +912,9 @@ impl ScheduleStage {
         });
 
         let to_next_stage = if let Some(to_next_stage) = maybe_to_next_stage {
-            to_next_stage
+            *to_next_stage
         } else {
-            &background_ee_reaper.unwrap().1
+            background_ee_reaper.unwrap().1
         };
 
         loop {

From eed54e6611a9f22bbf815f3df6f1ad8d51733db8 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 22 Aug 2022 12:43:42 +0900
Subject: [PATCH 0887/3199] save

---
 scheduler/src/lib.rs | 13 +++++--------
 1 file changed, 5 insertions(+), 8 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 3f82d8bbb6c2fb..91d172f97cf8ac 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -900,7 +900,10 @@ impl ScheduleStage {
     ) {
         let mut executing_queue_count = 0;
         let mut current_unique_key = u64::max_value();
-        let background_ee_reaper = maybe_to_next_stage.is_none().then(|| {
+
+        let to_next_stage = if let Some(to_next_stage) = maybe_to_next_stage {
+            (None, *to_next_stage)
+        } else {
             let (ee_sender, ee_receiver) = crossbeam_channel::unbounded::<Box<ExecutionEnvironment>>();
             let h = std::thread::Builder::new().name("sol-reaper".to_string()).spawn(move || {
                 while let a = ee_receiver.recv().unwrap() {
@@ -908,13 +911,7 @@ impl ScheduleStage {
                 }
             }).unwrap();
 
-            (h, ee_sender)
-        });
-
-        let to_next_stage = if let Some(to_next_stage) = maybe_to_next_stage {
-            *to_next_stage
-        } else {
-            background_ee_reaper.unwrap().1
+            (Some(h), ee_sender)
         };
 
         loop {

From 97b55903dafa7f99da583b61bca578388660b9a2 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 22 Aug 2022 12:43:59 +0900
Subject: [PATCH 0888/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 91d172f97cf8ac..31c3b259a181f9 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -901,7 +901,7 @@ impl ScheduleStage {
         let mut executing_queue_count = 0;
         let mut current_unique_key = u64::max_value();
 
-        let to_next_stage = if let Some(to_next_stage) = maybe_to_next_stage {
+        let (maybe_handle, to_next_stage) = if let Some(to_next_stage) = maybe_to_next_stage {
             (None, *to_next_stage)
         } else {
             let (ee_sender, ee_receiver) = crossbeam_channel::unbounded::<Box<ExecutionEnvironment>>();

From 98f4d6c340a050f3aea571fa05a58a2ecb026c40 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 22 Aug 2022 12:44:12 +0900
Subject: [PATCH 0889/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 31c3b259a181f9..fc6f5f0e730696 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -902,7 +902,7 @@ impl ScheduleStage {
         let mut current_unique_key = u64::max_value();
 
         let (maybe_handle, to_next_stage) = if let Some(to_next_stage) = maybe_to_next_stage {
-            (None, *to_next_stage)
+            (None, to_next_stage)
         } else {
             let (ee_sender, ee_receiver) = crossbeam_channel::unbounded::<Box<ExecutionEnvironment>>();
             let h = std::thread::Builder::new().name("sol-reaper".to_string()).spawn(move || {

From 4d1bdf3bdff96bdf0431c6cb8ef5dd25f7d249fa Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 22 Aug 2022 12:44:59 +0900
Subject: [PATCH 0890/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index fc6f5f0e730696..bd0ee49b9f86b7 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -911,7 +911,7 @@ impl ScheduleStage {
                 }
             }).unwrap();
 
-            (Some(h), ee_sender)
+            (Some(h, ee_sender, ee_receiver), &ee_sender)
         };
 
         loop {

From f80b3787ec07f8859b504a5e567d167b91f24553 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 22 Aug 2022 12:45:20 +0900
Subject: [PATCH 0891/3199] save

---
 scheduler/src/lib.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index bd0ee49b9f86b7..ad6003579512c0 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -902,7 +902,7 @@ impl ScheduleStage {
         let mut current_unique_key = u64::max_value();
 
         let (maybe_handle, to_next_stage) = if let Some(to_next_stage) = maybe_to_next_stage {
-            (None, to_next_stage)
+            (to_next_stage, None)
         } else {
             let (ee_sender, ee_receiver) = crossbeam_channel::unbounded::<Box<ExecutionEnvironment>>();
             let h = std::thread::Builder::new().name("sol-reaper".to_string()).spawn(move || {
@@ -911,7 +911,7 @@ impl ScheduleStage {
                 }
             }).unwrap();
 
-            (Some(h, ee_sender, ee_receiver), &ee_sender)
+            (&ee_sender, Some((h, ee_sender, ee_receiver)))
         };
 
         loop {

From 0ca0a78a0ab12e42b0ff2902644baf02d57e8cc2 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 22 Aug 2022 12:45:40 +0900
Subject: [PATCH 0892/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index ad6003579512c0..20a57ec2a70977 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -901,7 +901,7 @@ impl ScheduleStage {
         let mut executing_queue_count = 0;
         let mut current_unique_key = u64::max_value();
 
-        let (maybe_handle, to_next_stage) = if let Some(to_next_stage) = maybe_to_next_stage {
+        let (to_next_stage, maybe_jon_handle) = if let Some(to_next_stage) = maybe_to_next_stage {
             (to_next_stage, None)
         } else {
             let (ee_sender, ee_receiver) = crossbeam_channel::unbounded::<Box<ExecutionEnvironment>>();

From 52a7a3ce4e7fd29121753de9e42acbb95f7a03bd Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 22 Aug 2022 12:46:49 +0900
Subject: [PATCH 0893/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 20a57ec2a70977..7242d4504efe36 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -900,11 +900,11 @@ impl ScheduleStage {
     ) {
         let mut executing_queue_count = 0;
         let mut current_unique_key = u64::max_value();
+        let (ee_sender, ee_receiver) = crossbeam_channel::unbounded::<Box<ExecutionEnvironment>>();
 
         let (to_next_stage, maybe_jon_handle) = if let Some(to_next_stage) = maybe_to_next_stage {
             (to_next_stage, None)
         } else {
-            let (ee_sender, ee_receiver) = crossbeam_channel::unbounded::<Box<ExecutionEnvironment>>();
             let h = std::thread::Builder::new().name("sol-reaper".to_string()).spawn(move || {
                 while let a = ee_receiver.recv().unwrap() {
                     drop(a);

From 4cf65842286659ebf3e7e3ed8cba3a7193873929 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 22 Aug 2022 12:47:00 +0900
Subject: [PATCH 0894/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 7242d4504efe36..791fc84edaa98b 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -911,7 +911,7 @@ impl ScheduleStage {
                 }
             }).unwrap();
 
-            (&ee_sender, Some((h, ee_sender, ee_receiver)))
+            (&ee_sender, Some(h))
         };
 
         loop {

From 3471ce02c8ef265f8e7c4c23c9224c1a4cea4531 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 22 Aug 2022 12:53:41 +0900
Subject: [PATCH 0895/3199] save

---
 scheduler/src/lib.rs | 23 ++++++++++++++---------
 1 file changed, 14 insertions(+), 9 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 791fc84edaa98b..2fdfe819a41cdf 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -458,11 +458,11 @@ impl TaskQueue {
     fn entry_to_execute(
         &mut self,
         unique_weight: UniqueWeight,
-    ) -> TaskQueueOccupiedEntry<'_, UniqueWeight, TaskInQueue> {
+    ) -> Option<TaskQueueOccupiedEntry<'_, UniqueWeight, TaskInQueue>> {
         let queue_entry = self.tasks.entry(unique_weight);
         match queue_entry {
             TaskQueueEntry::Occupied(queue_entry) => queue_entry,
-            TaskQueueEntry::Vacant(_queue_entry) => unreachable!(),
+            TaskQueueEntry::Vacant(_queue_entry) => None,
         }
     }
 
@@ -476,7 +476,7 @@ impl TaskQueue {
     ) -> Option<TaskQueueOccupiedEntry<'_, UniqueWeight, TaskInQueue>> {
         //panic!()//self.tasks.last_entry()
         let k = self.tasks.get_max().map(|(k, _v)| *k);
-        k.map(|k| self.entry_to_execute(k))
+        k.map(|k| self.entry_to_execute(k).unwrap())
     }
 
     fn task_count(&self) -> usize {
@@ -612,7 +612,11 @@ impl ScheduleStage {
                 let uw = *weight_from_contended.key();
                 weight_from_contended.remove();
 
-                Some(( None, contended_queue.entry_to_execute(uw)))
+                if let Some (queue_entry) = contended_queue.entry_to_execute(uw) {
+                    Some(( None, queue_entry))
+                } else {
+                    continue;
+                }
             },
             (Some(heaviest_runnable_entry), Some(weight_from_contended)) => {
                 let weight_from_runnable = heaviest_runnable_entry.key();
@@ -625,10 +629,11 @@ impl ScheduleStage {
                     trace!("select: contended > runnnable");
                     let uw = *uw;
                     weight_from_contended.remove();
-                    Some((
-                        None,
-                        contended_queue.entry_to_execute(uw),
-                    ))
+                    if let Some (queue_entry) = contended_queue.entry_to_execute(uw) {
+                        Some(( None, queue_entry))
+                    } else {
+                        continue;
+                    }
                 } else {
                     unreachable!(
                         "identical unique weights shouldn't exist in both runnable and contended"
@@ -651,7 +656,7 @@ impl ScheduleStage {
     ) -> Option<(UniqueWeight, TaskInQueue)> {
         if let Some(a) = address_book.fulfilled_provisional_task_ids.pop_last() {
             trace!("expediate pop from provisional queue [rest: {}]", address_book.fulfilled_provisional_task_ids.len());
-            let queue_entry = contended_queue.entry_to_execute(a.0);
+            let queue_entry = contended_queue.entry_to_execute(a.0).unwrap();
             let mut task = queue_entry.remove();
             return Some((a.0, task));
         }

From 30cba6dec04db00439f6c4d50418189672269dfa Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 22 Aug 2022 12:54:24 +0900
Subject: [PATCH 0896/3199] save

---
 scheduler/src/lib.rs | 12 +++++++-----
 1 file changed, 7 insertions(+), 5 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 2fdfe819a41cdf..659a44a15b28e6 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -599,13 +599,14 @@ impl ScheduleStage {
         Option<&'a mut TaskQueue>,
         TaskQueueOccupiedEntry<'a, UniqueWeight, TaskInQueue>,
     )> {
+        loop {
         match (
             runnable_queue.heaviest_entry_to_execute(),
             Self::get_heaviest_from_contended(address_book),
         ) {
             (Some(heaviest_runnable_entry), None) => {
                 trace!("select: runnable only");
-                Some((Some(contended_queue), heaviest_runnable_entry))
+                return Some((Some(contended_queue), heaviest_runnable_entry))
             }
             (None, Some(weight_from_contended)) => {
                 trace!("select: contended only");
@@ -613,7 +614,7 @@ impl ScheduleStage {
                 weight_from_contended.remove();
 
                 if let Some (queue_entry) = contended_queue.entry_to_execute(uw) {
-                    Some(( None, queue_entry))
+                    return Some(( None, queue_entry))
                 } else {
                     continue;
                 }
@@ -624,13 +625,13 @@ impl ScheduleStage {
 
                 if true || weight_from_runnable > uw {
                     trace!("select: runnable > contended");
-                    Some((Some(contended_queue), heaviest_runnable_entry))
+                    return Some((Some(contended_queue), heaviest_runnable_entry))
                 } else if false && uw > weight_from_runnable {
                     trace!("select: contended > runnnable");
                     let uw = *uw;
                     weight_from_contended.remove();
                     if let Some (queue_entry) = contended_queue.entry_to_execute(uw) {
-                        Some(( None, queue_entry))
+                        return Some(( None, queue_entry))
                     } else {
                         continue;
                     }
@@ -642,9 +643,10 @@ impl ScheduleStage {
             }
             (None, None) => {
                 trace!("select: none");
-                None
+                return None
             }
         }
+        }
     }
 
     #[inline(never)]

From 427d883373574cab03332eea5ac02a905222134a Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 22 Aug 2022 12:54:41 +0900
Subject: [PATCH 0897/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 659a44a15b28e6..021007c3d3af75 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -461,7 +461,7 @@ impl TaskQueue {
     ) -> Option<TaskQueueOccupiedEntry<'_, UniqueWeight, TaskInQueue>> {
         let queue_entry = self.tasks.entry(unique_weight);
         match queue_entry {
-            TaskQueueEntry::Occupied(queue_entry) => queue_entry,
+            TaskQueueEntry::Occupied(queue_entry) => Some(queue_entry),
             TaskQueueEntry::Vacant(_queue_entry) => None,
         }
     }

From a3f89a1f80360b41230fb8065bcfff7fcb7103ed Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 22 Aug 2022 12:55:48 +0900
Subject: [PATCH 0898/3199] save

---
 scheduler/src/lib.rs | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 021007c3d3af75..f3b41fb0a49a95 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -599,7 +599,6 @@ impl ScheduleStage {
         Option<&'a mut TaskQueue>,
         TaskQueueOccupiedEntry<'a, UniqueWeight, TaskInQueue>,
     )> {
-        loop {
         match (
             runnable_queue.heaviest_entry_to_execute(),
             Self::get_heaviest_from_contended(address_book),
@@ -616,7 +615,7 @@ impl ScheduleStage {
                 if let Some (queue_entry) = contended_queue.entry_to_execute(uw) {
                     return Some(( None, queue_entry))
                 } else {
-                    continue;
+                    return None;
                 }
             },
             (Some(heaviest_runnable_entry), Some(weight_from_contended)) => {
@@ -633,7 +632,7 @@ impl ScheduleStage {
                     if let Some (queue_entry) = contended_queue.entry_to_execute(uw) {
                         return Some(( None, queue_entry))
                     } else {
-                        continue;
+                        return None;
                     }
                 } else {
                     unreachable!(
@@ -646,7 +645,6 @@ impl ScheduleStage {
                 return None
             }
         }
-        }
     }
 
     #[inline(never)]

From 11600cd7628c249a586cbf6d57ef8f0004250fec Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 22 Aug 2022 13:18:34 +0900
Subject: [PATCH 0899/3199] save

---
 scheduler/src/lib.rs | 19 ++++++++++++++++++-
 1 file changed, 18 insertions(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index f3b41fb0a49a95..6ab8b63052365a 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -213,7 +213,24 @@ pub struct AddressBook {
     book: AddressMap,
     uncontended_task_ids: WeightedTaskIds,
     fulfilled_provisional_task_ids: WeightedTaskIds,
-    provisioning_trackers: std::collections::HashMap<UniqueWeight, usize>, 
+    provisioning_trackers: std::collections::HashMap<UniqueWeight, ProvisioningTracker>, 
+}
+
+struct ProvisioningTracker {
+    remaining_count: usize,
+}
+
+impl ProvisioningTracker {
+    fn new(remaining_count) -> Self {
+        Self { remaining_count }
+    }
+
+    fn is_fulfilled(&self) -> bool {
+        self.remaining_count == 0
+    }
+
+    fn tick(&mut self) {
+    }
 }
 
 impl AddressBook {

From 8e16c0e22c9a67df294fce67777781d008989409 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 22 Aug 2022 13:18:58 +0900
Subject: [PATCH 0900/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 6ab8b63052365a..7a9deede95c69d 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -792,7 +792,7 @@ impl ScheduleStage {
                 }
             }
         }
-        address_book.provisioning_trackers.insert(*unique_weight, provisional_count);
+        address_book.provisioning_trackers.insert(*unique_weight, ProvisioningTracker::new(provisional_count));
         trace!("provisioning_trackers: {}", address_book.provisioning_trackers.len());
     }
 

From 420e7400e3e5fc095e9e414a263aced993b74ef4 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 22 Aug 2022 13:21:56 +0900
Subject: [PATCH 0901/3199] save

---
 scheduler/src/lib.rs | 15 ++++++++++-----
 1 file changed, 10 insertions(+), 5 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 7a9deede95c69d..a7c64984e2bc00 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -230,6 +230,11 @@ impl ProvisioningTracker {
     }
 
     fn tick(&mut self) {
+        self.remaining_count = self.remaining_count.checked_sub(1).unwrap();
+    }
+
+    fn count(&self) -> usize {
+        self.remaining_count
     }
 }
 
@@ -834,14 +839,14 @@ impl ScheduleStage {
                 for task_id in std::mem::take(&mut page.provisional_task_ids).keys() {
                     match address_book.provisioning_trackers.entry(*task_id) {
                         std::collections::hash_map::Entry::Occupied(mut tracker_entry) => {
-                            let count = tracker_entry.get_mut();
-                            *count = count.checked_sub(1).unwrap();
-                            if *count == 0 {
-                                trace!("provisional lock decrease: {} => {} (!)", *count + 1, *count);
+                            let tracker = tracker_entry.get_mut();
+                            tracker.tick();
+                            if tracker.is_fulfilled() {
+                                trace!("provisional lock decrease: {} => {} (!)", tracker.count() + 1, tracker.count());
                                 tracker_entry.remove();
                                 address_book.fulfilled_provisional_task_ids.insert(*task_id, ());
                             } else {
-                                trace!("provisional lock decrease: {} => {}", *count + 1, *count);
+                                trace!("provisional lock decrease: {} => {}", tracker.count() + 1, tracker.count();
                             }
                         },
                         std::collections::hash_map::Entry::Vacant(_) => {

From fcbeb79a9e86dbf1a0e0df7cfee93682eb9a733e Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 22 Aug 2022 13:22:09 +0900
Subject: [PATCH 0902/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index a7c64984e2bc00..3305f374039151 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -846,7 +846,7 @@ impl ScheduleStage {
                                 tracker_entry.remove();
                                 address_book.fulfilled_provisional_task_ids.insert(*task_id, ());
                             } else {
-                                trace!("provisional lock decrease: {} => {}", tracker.count() + 1, tracker.count();
+                                trace!("provisional lock decrease: {} => {}", tracker.count() + 1, tracker.count());
                             }
                         },
                         std::collections::hash_map::Entry::Vacant(_) => {

From c1f886644109fb61c9fb198cca20511608dceb6c Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 22 Aug 2022 13:22:24 +0900
Subject: [PATCH 0903/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 3305f374039151..51ad539dac0641 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -221,7 +221,7 @@ struct ProvisioningTracker {
 }
 
 impl ProvisioningTracker {
-    fn new(remaining_count) -> Self {
+    fn new(remaining_count: usize) -> Self {
         Self { remaining_count }
     }
 

From a921a3d36648770d7f5732f583ff8b45fc55c389 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 22 Aug 2022 13:23:50 +0900
Subject: [PATCH 0904/3199] save

---
 scheduler/src/lib.rs | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 51ad539dac0641..2203154b3c9a4d 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -233,6 +233,10 @@ impl ProvisioningTracker {
         self.remaining_count = self.remaining_count.checked_sub(1).unwrap();
     }
 
+    fn prev_count(&self) -> usize {
+        self.remaining_count + 1
+    }
+
     fn count(&self) -> usize {
         self.remaining_count
     }
@@ -842,11 +846,11 @@ impl ScheduleStage {
                             let tracker = tracker_entry.get_mut();
                             tracker.tick();
                             if tracker.is_fulfilled() {
-                                trace!("provisional lock decrease: {} => {} (!)", tracker.count() + 1, tracker.count());
+                                trace!("provisioning tracker tick: {} => {} (!)", tracker.prev_count(), tracker.count());
                                 tracker_entry.remove();
                                 address_book.fulfilled_provisional_task_ids.insert(*task_id, ());
                             } else {
-                                trace!("provisional lock decrease: {} => {}", tracker.count() + 1, tracker.count());
+                                trace!("provisioning tracker tick: {} => {}", tracker.prev_count(), tracker.count());
                             }
                         },
                         std::collections::hash_map::Entry::Vacant(_) => {

From 57d80be15952334cf31520627ad80c1d962d62ab Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 22 Aug 2022 13:27:52 +0900
Subject: [PATCH 0905/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 2203154b3c9a4d..f012e39ac68d2a 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -212,8 +212,8 @@ type AddressMapEntry<'a> = dashmap::mapref::entry::Entry<'a, Pubkey, PageRc>;
 pub struct AddressBook {
     book: AddressMap,
     uncontended_task_ids: WeightedTaskIds,
-    fulfilled_provisional_task_ids: WeightedTaskIds,
     provisioning_trackers: std::collections::HashMap<UniqueWeight, ProvisioningTracker>, 
+    fulfilled_provisional_task_ids: WeightedTaskIds,
 }
 
 struct ProvisioningTracker {

From febca9e65eba60bdebebbe3d1374e6ebb079f1bb Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 22 Aug 2022 13:29:15 +0900
Subject: [PATCH 0906/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index f012e39ac68d2a..b35fcb127d9948 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -873,7 +873,7 @@ impl ScheduleStage {
     ) -> Box<ExecutionEnvironment> {
         let mut rng = rand::thread_rng();
         // load account now from AccountsDb
-        let checkpointed_contended_queue = (*contended_queue).clone();
+        let checkpointed_contended_queue = contended_queue.clone();
 
         Box::new(ExecutionEnvironment {
             task,

From 73197f9fda2a9af16a0447f8d85bd9028c92cb51 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 22 Aug 2022 16:58:07 +0900
Subject: [PATCH 0907/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index b35fcb127d9948..925221558062d3 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -840,7 +840,7 @@ impl ScheduleStage {
             }
             if page.current_usage == Usage::Unused && page.next_usage != Usage::Unused {
                 page.switch_to_next_usage();
-                for task_id in std::mem::take(&mut page.provisional_task_ids).keys() {
+                for task_id in std::mem::take(&mut page.provisional_task_ids).keys().into_iter() {
                     match address_book.provisioning_trackers.entry(*task_id) {
                         std::collections::hash_map::Entry::Occupied(mut tracker_entry) => {
                             let tracker = tracker_entry.get_mut();

From b6e2f9508f71e531dfc9d731ee2324b037babd52 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 22 Aug 2022 16:58:23 +0900
Subject: [PATCH 0908/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 925221558062d3..3f6ec92ecc9ed0 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -841,7 +841,7 @@ impl ScheduleStage {
             if page.current_usage == Usage::Unused && page.next_usage != Usage::Unused {
                 page.switch_to_next_usage();
                 for task_id in std::mem::take(&mut page.provisional_task_ids).keys().into_iter() {
-                    match address_book.provisioning_trackers.entry(*task_id) {
+                    match address_book.provisioning_trackers.entry(task_id) {
                         std::collections::hash_map::Entry::Occupied(mut tracker_entry) => {
                             let tracker = tracker_entry.get_mut();
                             tracker.tick();

From 49b958457a82bd99fc7266d46b5a5ebaf8c869de Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 22 Aug 2022 17:00:25 +0900
Subject: [PATCH 0909/3199] save

---
 scheduler/src/lib.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 3f6ec92ecc9ed0..9584f85820f1da 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -178,7 +178,7 @@ pub struct Page {
     current_usage: Usage,
     next_usage: Usage,
     pub contended_unique_weights: TaskIds,
-    provisional_task_ids: WeightedTaskIds,
+    provisional_task_ids: std::collections::HashSet<TaskId>,
     //loaded account from Accounts db
     //comulative_cu for qos; i.e. track serialized cumulative keyed by addresses and bail out block
     //producing as soon as any one of cu from the executing thread reaches to the limit
@@ -841,7 +841,7 @@ impl ScheduleStage {
             if page.current_usage == Usage::Unused && page.next_usage != Usage::Unused {
                 page.switch_to_next_usage();
                 for task_id in std::mem::take(&mut page.provisional_task_ids).keys().into_iter() {
-                    match address_book.provisioning_trackers.entry(task_id) {
+                    match address_book.provisioning_trackers.entry(*task_id) {
                         std::collections::hash_map::Entry::Occupied(mut tracker_entry) => {
                             let tracker = tracker_entry.get_mut();
                             tracker.tick();

From 804d166170744797c375900addfeab109a4c86e2 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 22 Aug 2022 17:00:48 +0900
Subject: [PATCH 0910/3199] save

---
 scheduler/src/lib.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 9584f85820f1da..6dd88ba2a27f99 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -791,7 +791,7 @@ impl ScheduleStage {
             //AddressBook::forget_address_contention(&unique_weight, &mut l);
             match l.status {
                 LockStatus::Provisional => {
-                    l.target.page_mut().provisional_task_ids.insert(*unique_weight, ());
+                    l.target.page_mut().provisional_task_ids.insert(*unique_weight);
                 }
                 LockStatus::Succeded => {
                     // do nothing
@@ -840,7 +840,7 @@ impl ScheduleStage {
             }
             if page.current_usage == Usage::Unused && page.next_usage != Usage::Unused {
                 page.switch_to_next_usage();
-                for task_id in std::mem::take(&mut page.provisional_task_ids).keys().into_iter() {
+                for task_id in std::mem::take(&mut page.provisional_task_ids).into_iter() {
                     match address_book.provisioning_trackers.entry(*task_id) {
                         std::collections::hash_map::Entry::Occupied(mut tracker_entry) => {
                             let tracker = tracker_entry.get_mut();

From 70293f6379ec699f59acf306e2156cd84b5dafb3 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 22 Aug 2022 17:01:09 +0900
Subject: [PATCH 0911/3199] save

---
 scheduler/src/lib.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 6dd88ba2a27f99..b185962ba9f4ba 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -841,14 +841,14 @@ impl ScheduleStage {
             if page.current_usage == Usage::Unused && page.next_usage != Usage::Unused {
                 page.switch_to_next_usage();
                 for task_id in std::mem::take(&mut page.provisional_task_ids).into_iter() {
-                    match address_book.provisioning_trackers.entry(*task_id) {
+                    match address_book.provisioning_trackers.entry(task_id) {
                         std::collections::hash_map::Entry::Occupied(mut tracker_entry) => {
                             let tracker = tracker_entry.get_mut();
                             tracker.tick();
                             if tracker.is_fulfilled() {
                                 trace!("provisioning tracker tick: {} => {} (!)", tracker.prev_count(), tracker.count());
                                 tracker_entry.remove();
-                                address_book.fulfilled_provisional_task_ids.insert(*task_id, ());
+                                address_book.fulfilled_provisional_task_ids.insert(task_id, ());
                             } else {
                                 trace!("provisioning tracker tick: {} => {}", tracker.prev_count(), tracker.count());
                             }

From 62c868ed78ee2ae1309633248b153633cf8dbdcd Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 22 Aug 2022 17:13:28 +0900
Subject: [PATCH 0912/3199] save

---
 scheduler/src/lib.rs | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index b185962ba9f4ba..42acf4999de411 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -465,7 +465,8 @@ pub struct Task {
 #[derive(Default, Debug, Clone)]
 pub struct TaskQueue {
     //tasks: std::collections::BTreeMap<UniqueWeight, Task>,
-    tasks: im::OrdMap<UniqueWeight, TaskInQueue>,
+    //tasks: im::OrdMap<UniqueWeight, TaskInQueue>,
+    tasks: im::Map<UniqueWeight, TaskInQueue>,
 }
 
 type TaskInQueue = std::sync::Arc<Task>;

From 056d7ac86bec52e12bfe04aeaf9fdf9d32b870f9 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 22 Aug 2022 17:15:21 +0900
Subject: [PATCH 0913/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 42acf4999de411..47ceca78b357c7 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -466,7 +466,7 @@ pub struct Task {
 pub struct TaskQueue {
     //tasks: std::collections::BTreeMap<UniqueWeight, Task>,
     //tasks: im::OrdMap<UniqueWeight, TaskInQueue>,
-    tasks: im::Map<UniqueWeight, TaskInQueue>,
+    tasks: im::HashMap<UniqueWeight, TaskInQueue>,
 }
 
 type TaskInQueue = std::sync::Arc<Task>;

From 90716838309efafceb8bf09f25fd89a92d922e2c Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 22 Aug 2022 17:15:46 +0900
Subject: [PATCH 0914/3199] save

---
 scheduler/src/lib.rs | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 47ceca78b357c7..700c5a5b84f31d 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -470,8 +470,10 @@ pub struct TaskQueue {
 }
 
 type TaskInQueue = std::sync::Arc<Task>;
-type TaskQueueEntry<'a, K, V> = im::ordmap::Entry<'a, K, V>;
-type TaskQueueOccupiedEntry<'a, K, V> = im::ordmap::OccupiedEntry<'a, K, V>;
+//type TaskQueueEntry<'a, K, V> = im::ordmap::Entry<'a, K, V>;
+//type TaskQueueOccupiedEntry<'a, K, V> = im::ordmap::OccupiedEntry<'a, K, V>;
+type TaskQueueEntry<'a, K, V> = im::hashmap::Entry<'a, K, V>;
+type TaskQueueOccupiedEntry<'a, K, V> = im::hashmap::OccupiedEntry<'a, K, V>;
 
 impl TaskQueue {
     #[inline(never)]

From eb55558d089682f5836250d06be94d385591633f Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 22 Aug 2022 17:16:04 +0900
Subject: [PATCH 0915/3199] save

---
 scheduler/src/lib.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 700c5a5b84f31d..e7f5b48726a88e 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -472,8 +472,8 @@ pub struct TaskQueue {
 type TaskInQueue = std::sync::Arc<Task>;
 //type TaskQueueEntry<'a, K, V> = im::ordmap::Entry<'a, K, V>;
 //type TaskQueueOccupiedEntry<'a, K, V> = im::ordmap::OccupiedEntry<'a, K, V>;
-type TaskQueueEntry<'a, K, V> = im::hashmap::Entry<'a, K, V>;
-type TaskQueueOccupiedEntry<'a, K, V> = im::hashmap::OccupiedEntry<'a, K, V>;
+type TaskQueueEntry<'a, K, V, S> = im::hashmap::Entry<'a, K, V, S>;
+type TaskQueueOccupiedEntry<'a, K, V, S> = im::hashmap::OccupiedEntry<'a, K, V, S>;
 
 impl TaskQueue {
     #[inline(never)]

From 624748cbb2d0d87785ebf32dbbca6c453186e33a Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 22 Aug 2022 17:16:29 +0900
Subject: [PATCH 0916/3199] save

---
 scheduler/src/lib.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index e7f5b48726a88e..40c409b867fd79 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -502,7 +502,7 @@ impl TaskQueue {
     #[inline(never)]
     fn heaviest_entry_to_execute(
         &mut self,
-    ) -> Option<TaskQueueOccupiedEntry<'_, UniqueWeight, TaskInQueue>> {
+    ) -> Option<TaskQueueOccupiedEntry<'_, UniqueWeight, TaskInQueue, _>> {
         //panic!()//self.tasks.last_entry()
         let k = self.tasks.get_max().map(|(k, _v)| *k);
         k.map(|k| self.entry_to_execute(k).unwrap())
@@ -626,7 +626,7 @@ impl ScheduleStage {
         address_book: &mut AddressBook,
     ) -> Option<(
         Option<&'a mut TaskQueue>,
-        TaskQueueOccupiedEntry<'a, UniqueWeight, TaskInQueue>,
+        TaskQueueOccupiedEntry<'a, UniqueWeight, TaskInQueue, _>,
     )> {
         match (
             runnable_queue.heaviest_entry_to_execute(),

From 24f612bc30e319902759ecfc4ff5c7c287efa684 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 22 Aug 2022 17:16:39 +0900
Subject: [PATCH 0917/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 40c409b867fd79..05475926bf059e 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -487,7 +487,7 @@ impl TaskQueue {
     fn entry_to_execute(
         &mut self,
         unique_weight: UniqueWeight,
-    ) -> Option<TaskQueueOccupiedEntry<'_, UniqueWeight, TaskInQueue>> {
+    ) -> Option<TaskQueueOccupiedEntry<'_, UniqueWeight, TaskInQueue, _>> {
         let queue_entry = self.tasks.entry(unique_weight);
         match queue_entry {
             TaskQueueEntry::Occupied(queue_entry) => Some(queue_entry),

From ac09e0a1f23c93a22cace6d0884ca5e389cb1c7b Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 22 Aug 2022 17:16:56 +0900
Subject: [PATCH 0918/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 05475926bf059e..d52e3ec743b72d 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -473,7 +473,7 @@ type TaskInQueue = std::sync::Arc<Task>;
 //type TaskQueueEntry<'a, K, V> = im::ordmap::Entry<'a, K, V>;
 //type TaskQueueOccupiedEntry<'a, K, V> = im::ordmap::OccupiedEntry<'a, K, V>;
 type TaskQueueEntry<'a, K, V, S> = im::hashmap::Entry<'a, K, V, S>;
-type TaskQueueOccupiedEntry<'a, K, V, S> = im::hashmap::OccupiedEntry<'a, K, V, S>;
+type TaskQueueOccupiedEntry<'a, K, V> = im::hashmap::OccupiedEntry<'a, K, V, RandomState>;
 
 impl TaskQueue {
     #[inline(never)]

From 9006e92519c66cdaac9fe548320ad32b982ffca4 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 22 Aug 2022 17:17:21 +0900
Subject: [PATCH 0919/3199] save

---
 scheduler/src/lib.rs | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index d52e3ec743b72d..5c360b8b68572b 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -487,7 +487,7 @@ impl TaskQueue {
     fn entry_to_execute(
         &mut self,
         unique_weight: UniqueWeight,
-    ) -> Option<TaskQueueOccupiedEntry<'_, UniqueWeight, TaskInQueue, _>> {
+    ) -> Option<TaskQueueOccupiedEntry<'_, UniqueWeight, TaskInQueue>> {
         let queue_entry = self.tasks.entry(unique_weight);
         match queue_entry {
             TaskQueueEntry::Occupied(queue_entry) => Some(queue_entry),
@@ -502,7 +502,7 @@ impl TaskQueue {
     #[inline(never)]
     fn heaviest_entry_to_execute(
         &mut self,
-    ) -> Option<TaskQueueOccupiedEntry<'_, UniqueWeight, TaskInQueue, _>> {
+    ) -> Option<TaskQueueOccupiedEntry<'_, UniqueWeight, TaskInQueue>> {
         //panic!()//self.tasks.last_entry()
         let k = self.tasks.get_max().map(|(k, _v)| *k);
         k.map(|k| self.entry_to_execute(k).unwrap())
@@ -626,7 +626,7 @@ impl ScheduleStage {
         address_book: &mut AddressBook,
     ) -> Option<(
         Option<&'a mut TaskQueue>,
-        TaskQueueOccupiedEntry<'a, UniqueWeight, TaskInQueue, _>,
+        TaskQueueOccupiedEntry<'a, UniqueWeight, TaskInQueue>,
     )> {
         match (
             runnable_queue.heaviest_entry_to_execute(),

From 8b53b6d3c91f631ab730e7bc8a622642732266c5 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 22 Aug 2022 17:17:47 +0900
Subject: [PATCH 0920/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 5c360b8b68572b..a27d3199d717cc 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -473,7 +473,7 @@ type TaskInQueue = std::sync::Arc<Task>;
 //type TaskQueueEntry<'a, K, V> = im::ordmap::Entry<'a, K, V>;
 //type TaskQueueOccupiedEntry<'a, K, V> = im::ordmap::OccupiedEntry<'a, K, V>;
 type TaskQueueEntry<'a, K, V, S> = im::hashmap::Entry<'a, K, V, S>;
-type TaskQueueOccupiedEntry<'a, K, V> = im::hashmap::OccupiedEntry<'a, K, V, RandomState>;
+type TaskQueueOccupiedEntry<'a, K, V> = im::hashmap::OccupiedEntry<'a, K, V, std::collections::hash_map::RandomState>;
 
 impl TaskQueue {
     #[inline(never)]

From b99aab061c6224beba9077abda89dab0b3edd69d Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 22 Aug 2022 17:18:27 +0900
Subject: [PATCH 0921/3199] save

---
 scheduler/src/lib.rs | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index a27d3199d717cc..dd2c737852b9a3 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -504,7 +504,8 @@ impl TaskQueue {
         &mut self,
     ) -> Option<TaskQueueOccupiedEntry<'_, UniqueWeight, TaskInQueue>> {
         //panic!()//self.tasks.last_entry()
-        let k = self.tasks.get_max().map(|(k, _v)| *k);
+        //let k = self.tasks.get_max().map(|(k, _v)| *k);
+        let k = self.tasks.iter().next().map(|(k, _v)| *k);
         k.map(|k| self.entry_to_execute(k).unwrap())
     }
 

From a3ad4fd4ff8d02b52de14560fd45cf51f60beb3a Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 22 Aug 2022 17:22:09 +0900
Subject: [PATCH 0922/3199] save

---
 scheduler/src/lib.rs | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index dd2c737852b9a3..d78883894feabd 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -473,7 +473,7 @@ type TaskInQueue = std::sync::Arc<Task>;
 //type TaskQueueEntry<'a, K, V> = im::ordmap::Entry<'a, K, V>;
 //type TaskQueueOccupiedEntry<'a, K, V> = im::ordmap::OccupiedEntry<'a, K, V>;
 type TaskQueueEntry<'a, K, V, S> = im::hashmap::Entry<'a, K, V, S>;
-type TaskQueueOccupiedEntry<'a, K, V> = im::hashmap::OccupiedEntry<'a, K, V, std::collections::hash_map::RandomState>;
+type TaskQueueOccupiedEntry<'a> = im::hashmap::OccupiedEntry<'a, UniqueWeight, TaskQueue, std::collections::hash_map::RandomState>;
 
 impl TaskQueue {
     #[inline(never)]
@@ -487,7 +487,7 @@ impl TaskQueue {
     fn entry_to_execute(
         &mut self,
         unique_weight: UniqueWeight,
-    ) -> Option<TaskQueueOccupiedEntry<'_, UniqueWeight, TaskInQueue>> {
+    ) -> Option<TaskQueueOccupiedEntry<'_>> {
         let queue_entry = self.tasks.entry(unique_weight);
         match queue_entry {
             TaskQueueEntry::Occupied(queue_entry) => Some(queue_entry),
@@ -502,10 +502,10 @@ impl TaskQueue {
     #[inline(never)]
     fn heaviest_entry_to_execute(
         &mut self,
-    ) -> Option<TaskQueueOccupiedEntry<'_, UniqueWeight, TaskInQueue>> {
+    ) -> Option<TaskQueueOccupiedEntry<'_>> {
         //panic!()//self.tasks.last_entry()
-        //let k = self.tasks.get_max().map(|(k, _v)| *k);
-        let k = self.tasks.iter().next().map(|(k, _v)| *k);
+        let k = self.tasks.get_max().map(|(k, _v)| *k);
+        //let k = self.tasks.iter().next().map(|(k, _v)| *k);
         k.map(|k| self.entry_to_execute(k).unwrap())
     }
 
@@ -627,7 +627,7 @@ impl ScheduleStage {
         address_book: &mut AddressBook,
     ) -> Option<(
         Option<&'a mut TaskQueue>,
-        TaskQueueOccupiedEntry<'a, UniqueWeight, TaskInQueue>,
+        TaskQueueOccupiedEntry<'a>,
     )> {
         match (
             runnable_queue.heaviest_entry_to_execute(),

From bed2df0d13430013813cbb20c32274ece92335ba Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 22 Aug 2022 17:22:26 +0900
Subject: [PATCH 0923/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index d78883894feabd..6a762b4d455a7d 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -473,7 +473,7 @@ type TaskInQueue = std::sync::Arc<Task>;
 //type TaskQueueEntry<'a, K, V> = im::ordmap::Entry<'a, K, V>;
 //type TaskQueueOccupiedEntry<'a, K, V> = im::ordmap::OccupiedEntry<'a, K, V>;
 type TaskQueueEntry<'a, K, V, S> = im::hashmap::Entry<'a, K, V, S>;
-type TaskQueueOccupiedEntry<'a> = im::hashmap::OccupiedEntry<'a, UniqueWeight, TaskQueue, std::collections::hash_map::RandomState>;
+type TaskQueueOccupiedEntry<'a> = im::hashmap::OccupiedEntry<'a, UniqueWeight, TaskInQueue, std::collections::hash_map::RandomState>;
 
 impl TaskQueue {
     #[inline(never)]

From 305162b747761fcb7498037108631a445ada5267 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 22 Aug 2022 17:22:58 +0900
Subject: [PATCH 0924/3199] save

---
 scheduler/src/lib.rs | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 6a762b4d455a7d..5da4cda281cb11 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -465,15 +465,15 @@ pub struct Task {
 #[derive(Default, Debug, Clone)]
 pub struct TaskQueue {
     //tasks: std::collections::BTreeMap<UniqueWeight, Task>,
-    //tasks: im::OrdMap<UniqueWeight, TaskInQueue>,
-    tasks: im::HashMap<UniqueWeight, TaskInQueue>,
+    tasks: im::OrdMap<UniqueWeight, TaskInQueue>,
+    //tasks: im::HashMap<UniqueWeight, TaskInQueue>,
 }
 
 type TaskInQueue = std::sync::Arc<Task>;
-//type TaskQueueEntry<'a, K, V> = im::ordmap::Entry<'a, K, V>;
-//type TaskQueueOccupiedEntry<'a, K, V> = im::ordmap::OccupiedEntry<'a, K, V>;
-type TaskQueueEntry<'a, K, V, S> = im::hashmap::Entry<'a, K, V, S>;
-type TaskQueueOccupiedEntry<'a> = im::hashmap::OccupiedEntry<'a, UniqueWeight, TaskInQueue, std::collections::hash_map::RandomState>;
+type TaskQueueEntry<'a, K, V> = im::ordmap::Entry<'a, K, V>;
+type TaskQueueOccupiedEntry<'a> = im::ordmap::OccupiedEntry<'a, UniqueWeight, TaskInQueue>;
+//type TaskQueueEntry<'a, K, V, S> = im::hashmap::Entry<'a, K, V, S>;
+//type TaskQueueOccupiedEntry<'a> = im::hashmap::OccupiedEntry<'a, UniqueWeight, TaskInQueue, std::collections::hash_map::RandomState>;
 
 impl TaskQueue {
     #[inline(never)]

From c99c7e7c57833b460d80fc24db9c3c06f227ddd2 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 22 Aug 2022 22:39:49 +0900
Subject: [PATCH 0925/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 5da4cda281cb11..e0b12998b68beb 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -470,7 +470,7 @@ pub struct TaskQueue {
 }
 
 type TaskInQueue = std::sync::Arc<Task>;
-type TaskQueueEntry<'a, K, V> = im::ordmap::Entry<'a, K, V>;
+type TaskQueueEntry<'a> = im::ordmap::Entry<'a, UniqueWeight, TaskInQueue>;
 type TaskQueueOccupiedEntry<'a> = im::ordmap::OccupiedEntry<'a, UniqueWeight, TaskInQueue>;
 //type TaskQueueEntry<'a, K, V, S> = im::hashmap::Entry<'a, K, V, S>;
 //type TaskQueueOccupiedEntry<'a> = im::hashmap::OccupiedEntry<'a, UniqueWeight, TaskInQueue, std::collections::hash_map::RandomState>;

From ef9324834f577d3982a603136b3fd5a5bc96c077 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 22 Aug 2022 22:40:39 +0900
Subject: [PATCH 0926/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index e0b12998b68beb..ec539bab8232f6 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -472,7 +472,7 @@ pub struct TaskQueue {
 type TaskInQueue = std::sync::Arc<Task>;
 type TaskQueueEntry<'a> = im::ordmap::Entry<'a, UniqueWeight, TaskInQueue>;
 type TaskQueueOccupiedEntry<'a> = im::ordmap::OccupiedEntry<'a, UniqueWeight, TaskInQueue>;
-//type TaskQueueEntry<'a, K, V, S> = im::hashmap::Entry<'a, K, V, S>;
+//type TaskQueueEntry<'a> = im::hashmap::Entry<'a, UniqueWeight, TaskInQueue, , std::collections::hash_map::RandomState>;
 //type TaskQueueOccupiedEntry<'a> = im::hashmap::OccupiedEntry<'a, UniqueWeight, TaskInQueue, std::collections::hash_map::RandomState>;
 
 impl TaskQueue {

From be69683eafaf1f90d6d8f4a1195270071285d910 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 22 Aug 2022 23:02:01 +0900
Subject: [PATCH 0927/3199] save

---
 scheduler/src/lib.rs | 1 +
 1 file changed, 1 insertion(+)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index ec539bab8232f6..542e56214cae48 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -495,6 +495,7 @@ impl TaskQueue {
         }
     }
 
+    #[inline(never)]
     pub fn has_task(&self, unique_weight: &UniqueWeight) -> bool {
         self.tasks.contains_key(unique_weight)
     }

From dcd3ba63d769dc38b4229c54a7fbb24465f10cae Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 22 Aug 2022 23:12:21 +0900
Subject: [PATCH 0928/3199] save

---
 ledger-tool/src/main.rs | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/ledger-tool/src/main.rs b/ledger-tool/src/main.rs
index e8840b2b5fcd00..44506f624f6b13 100644
--- a/ledger-tool/src/main.rs
+++ b/ledger-tool/src/main.rs
@@ -387,6 +387,10 @@ fn output_slot(
                     std::thread::sleep(std::time::Duration::from_micros(10));
                 }
 
+                for lock_attempt in tx.1.iter() {
+                    lock_attempt.target.page_ref().contended_unique_weights.insert_task_id(weight);
+                }
+
                 muxed_sender
                     .send(solana_scheduler::Multiplexed::FromPrevious((
                         //Weight { ix: weight },

From 6cff8c14f12fe56c3560beefc7abcdddc3dc9baf Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 22 Aug 2022 23:14:10 +0900
Subject: [PATCH 0929/3199] save

---
 ledger-tool/src/main.rs | 19 +++++++++++++++++++
 1 file changed, 19 insertions(+)

diff --git a/ledger-tool/src/main.rs b/ledger-tool/src/main.rs
index 44506f624f6b13..5ac9eeed399bc4 100644
--- a/ledger-tool/src/main.rs
+++ b/ledger-tool/src/main.rs
@@ -334,6 +334,25 @@ fn output_slot(
                             .send(solana_scheduler::Multiplexed::FromExecute(ee))
                             .unwrap();
                         */
+                        let uq = ee.unique_weight;
+                        for lock_attempt in ee.task.tx.1.iter() {
+                            let page = lock_attempt.target.page_ref();
+                            page.contended_unique_weights.remove_task_id(&uq);
+                            if let Some(mut task_cursor) = page.contended_unique_weights.heaviest_task_cursor() {
+                                let mut found = true;
+                                while !ee.checkpointed_contended_queue.has_task(task_cursor.value()) {
+                                    if let Some(new_cursor) = task_cursor.prev() {
+                                        task_cursor = new_cursor;
+                                    } else {
+                                        found = false;
+                                        break;
+                                    }
+                                }
+                                if found {
+                                    lock_attempt.heaviest_uncontended.store(*(task_cursor.value()), std::sync::atomic::Ordering::SeqCst);
+                                }
+                            }
+                        }
                         post_execute_env_sender.send(ee).unwrap();
                     }
                 })

From 15f8f684204367b389238d830ce087f79d644a9a Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 23 Aug 2022 09:57:23 +0900
Subject: [PATCH 0930/3199] save

---
 scheduler/src/lib.rs | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 542e56214cae48..b25a1a97d3af09 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -459,6 +459,7 @@ struct Bundle {
 pub struct Task {
     pub tx: Box<(SanitizedTransaction, Vec<LockAttempt>)>, // actually should be Bundle
     pub contention_count: usize,
+    pub uncontended: std::sync::atomic::AtomicBool,
 }
 
 // RunnableQueue, ContendedQueue?
@@ -879,6 +880,7 @@ impl ScheduleStage {
         let mut rng = rand::thread_rng();
         // load account now from AccountsDb
         let checkpointed_contended_queue = contended_queue.clone();
+        task.uncontended.store(true, std::sync::atomic::Ordering::SeqCst);
 
         Box::new(ExecutionEnvironment {
             task,

From 785f8498085fefdd97ee63484a22853fb91aec10 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 23 Aug 2022 09:57:48 +0900
Subject: [PATCH 0931/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index b25a1a97d3af09..3f0a9f11e70f0f 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -597,7 +597,7 @@ impl ScheduleStage {
             },
             */
             unique_weight,
-            TaskInQueue::new(Task { tx, contention_count: 0 }),
+            TaskInQueue::new(Task { tx, contention_count: 0, uncontended: Default::default() }),
         );
         *unique_key -= 1;
     }

From b84cd60eceb83663717b9e2c68bebe6050153b63 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 23 Aug 2022 09:58:55 +0900
Subject: [PATCH 0932/3199] save

---
 scheduler/src/lib.rs | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 3f0a9f11e70f0f..44eae66de29173 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -498,7 +498,11 @@ impl TaskQueue {
 
     #[inline(never)]
     pub fn has_task(&self, unique_weight: &UniqueWeight) -> bool {
-        self.tasks.contains_key(unique_weight)
+        let queue_entry = self.tasks.entry(unique_weight);
+        match queue_entry {
+            TaskQueueEntry::Occupied(queue_entry) => queue_entry.value().uncontended.load(std::sync::atomic::Ordering::SeqCst),
+            TaskQueueEntry::Vacant(_queue_entry) => false,
+        }
     }
 
     #[inline(never)]

From 65130f86898c3bd148ce9ee255b1d0775c4f1eab Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 23 Aug 2022 09:59:07 +0900
Subject: [PATCH 0933/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 44eae66de29173..b6f10cda3c49a7 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -498,7 +498,7 @@ impl TaskQueue {
 
     #[inline(never)]
     pub fn has_task(&self, unique_weight: &UniqueWeight) -> bool {
-        let queue_entry = self.tasks.entry(unique_weight);
+        let queue_entry = self.tasks.entry(*unique_weight);
         match queue_entry {
             TaskQueueEntry::Occupied(queue_entry) => queue_entry.value().uncontended.load(std::sync::atomic::Ordering::SeqCst),
             TaskQueueEntry::Vacant(_queue_entry) => false,

From f84cce5ef300df1e383849014314f37291f64915 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 23 Aug 2022 09:59:35 +0900
Subject: [PATCH 0934/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index b6f10cda3c49a7..841e625f86a2f6 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -500,7 +500,7 @@ impl TaskQueue {
     pub fn has_task(&self, unique_weight: &UniqueWeight) -> bool {
         let queue_entry = self.tasks.entry(*unique_weight);
         match queue_entry {
-            TaskQueueEntry::Occupied(queue_entry) => queue_entry.value().uncontended.load(std::sync::atomic::Ordering::SeqCst),
+            TaskQueueEntry::Occupied(queue_entry) => queue_entry.get().uncontended.load(std::sync::atomic::Ordering::SeqCst),
             TaskQueueEntry::Vacant(_queue_entry) => false,
         }
     }

From 41062b7a7520f0f20cd361b12475d35b813c8f9e Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 23 Aug 2022 10:00:02 +0900
Subject: [PATCH 0935/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 841e625f86a2f6..3b41de7a3ca94c 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -498,7 +498,7 @@ impl TaskQueue {
 
     #[inline(never)]
     pub fn has_task(&self, unique_weight: &UniqueWeight) -> bool {
-        let queue_entry = self.tasks.entry(*unique_weight);
+        let queue_entry = self.tasks.get(*unique_weight);
         match queue_entry {
             TaskQueueEntry::Occupied(queue_entry) => queue_entry.get().uncontended.load(std::sync::atomic::Ordering::SeqCst),
             TaskQueueEntry::Vacant(_queue_entry) => false,

From b4980f591ee7a50118bbc4d6ac46c4d29e8f17af Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 23 Aug 2022 10:00:33 +0900
Subject: [PATCH 0936/3199] save

---
 scheduler/src/lib.rs | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 3b41de7a3ca94c..c649092755cc4f 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -498,10 +498,10 @@ impl TaskQueue {
 
     #[inline(never)]
     pub fn has_task(&self, unique_weight: &UniqueWeight) -> bool {
-        let queue_entry = self.tasks.get(*unique_weight);
-        match queue_entry {
-            TaskQueueEntry::Occupied(queue_entry) => queue_entry.get().uncontended.load(std::sync::atomic::Ordering::SeqCst),
-            TaskQueueEntry::Vacant(_queue_entry) => false,
+        let maybe_task = self.tasks.get(*unique_weight);
+        match maybe_task {
+            Some(task) => task.uncontended.load(std::sync::atomic::Ordering::SeqCst),
+            None => false,
         }
     }
 

From 4b027a077c3ef48e0dc1efeb0b4bbb7099862939 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 23 Aug 2022 10:00:44 +0900
Subject: [PATCH 0937/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index c649092755cc4f..85816d3674f6ac 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -498,7 +498,7 @@ impl TaskQueue {
 
     #[inline(never)]
     pub fn has_task(&self, unique_weight: &UniqueWeight) -> bool {
-        let maybe_task = self.tasks.get(*unique_weight);
+        let maybe_task = self.tasks.get(unique_weight);
         match maybe_task {
             Some(task) => task.uncontended.load(std::sync::atomic::Ordering::SeqCst),
             None => false,

From 7fad15f702325db36415672c3f8a481e89a4bd71 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 23 Aug 2022 10:02:17 +0900
Subject: [PATCH 0938/3199] save

---
 scheduler/src/lib.rs | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 85816d3674f6ac..5d4cde9b8daec0 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -462,6 +462,12 @@ pub struct Task {
     pub uncontended: std::sync::atomic::AtomicBool,
 }
 
+impl Task {
+    fn still_contended(&self) -> bool {
+        !self.uncontended.load(std::sync::atomic::Ordering::SeqCst)
+    }
+}
+
 // RunnableQueue, ContendedQueue?
 #[derive(Default, Debug, Clone)]
 pub struct TaskQueue {
@@ -500,7 +506,7 @@ impl TaskQueue {
     pub fn has_task(&self, unique_weight: &UniqueWeight) -> bool {
         let maybe_task = self.tasks.get(unique_weight);
         match maybe_task {
-            Some(task) => task.uncontended.load(std::sync::atomic::Ordering::SeqCst),
+            Some(task) => task.still_contended(),
             None => false,
         }
     }

From 6d68f8b6734753d26db6a2996ca92aacebeb8795 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 23 Aug 2022 10:04:25 +0900
Subject: [PATCH 0939/3199] save

---
 scheduler/src/lib.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 5d4cde9b8daec0..0feb8849010a06 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -657,7 +657,7 @@ impl ScheduleStage {
                 if let Some (queue_entry) = contended_queue.entry_to_execute(uw) {
                     return Some(( None, queue_entry))
                 } else {
-                    return None;
+                    unreachable!();
                 }
             },
             (Some(heaviest_runnable_entry), Some(weight_from_contended)) => {
@@ -674,7 +674,7 @@ impl ScheduleStage {
                     if let Some (queue_entry) = contended_queue.entry_to_execute(uw) {
                         return Some(( None, queue_entry))
                     } else {
-                        return None;
+                        unreachable!();
                     }
                 } else {
                     unreachable!(

From 05ba63f0f0fe2b16f471319accd60f51a440f245 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 23 Aug 2022 10:06:34 +0900
Subject: [PATCH 0940/3199] save

---
 scheduler/src/lib.rs | 1 +
 1 file changed, 1 insertion(+)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 0feb8849010a06..0eb31044210333 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -657,6 +657,7 @@ impl ScheduleStage {
                 if let Some (queue_entry) = contended_queue.entry_to_execute(uw) {
                     return Some(( None, queue_entry))
                 } else {
+                    dbg!(&uw);
                     unreachable!();
                 }
             },

From 694dc554872ec16bdfc100f0227373c75e2899a1 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 23 Aug 2022 10:11:26 +0900
Subject: [PATCH 0941/3199] save

---
 scheduler/src/lib.rs | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 0eb31044210333..71478622123399 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -154,7 +154,7 @@ impl TaskIds {
     }
 
     #[inline(never)]
-    fn has_task_ids(&self) -> bool {
+    fn has_more(&self) -> bool {
         !self.task_ids.is_empty()
     }
 
@@ -380,7 +380,7 @@ impl AddressBook {
 
         if newly_uncontended {
             page.current_usage = Usage::Unused;
-            if page.contended_unique_weights.has_task_ids() {
+            if page.contended_unique_weights.has_more() {
                 still_queued = true;
             }
         }
@@ -657,7 +657,6 @@ impl ScheduleStage {
                 if let Some (queue_entry) = contended_queue.entry_to_execute(uw) {
                     return Some(( None, queue_entry))
                 } else {
-                    dbg!(&uw);
                     unreachable!();
                 }
             },
@@ -852,7 +851,9 @@ impl ScheduleStage {
             if newly_uncontended_while_queued && page.next_usage == Usage::Unused {
                 let task_id = l.heaviest_uncontended.load(std::sync::atomic::Ordering::SeqCst);
                 if task_id != 0 {
-                    address_book.uncontended_task_ids.insert(task_id, ());
+                    if contended_queue.has_task(task_id) {
+                        address_book.uncontended_task_ids.insert(task_id, ());
+                    }
                 }
             }
             if page.current_usage == Usage::Unused && page.next_usage != Usage::Unused {

From 6590008aa0398dc41523a28e987394b74800a087 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 23 Aug 2022 10:11:33 +0900
Subject: [PATCH 0942/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 71478622123399..823385bce2f358 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -851,7 +851,7 @@ impl ScheduleStage {
             if newly_uncontended_while_queued && page.next_usage == Usage::Unused {
                 let task_id = l.heaviest_uncontended.load(std::sync::atomic::Ordering::SeqCst);
                 if task_id != 0 {
-                    if contended_queue.has_task(task_id) {
+                    if contended_queue.has_task(&task_id) {
                         address_book.uncontended_task_ids.insert(task_id, ());
                     }
                 }

From cc87265dfd78cb1614a4a0165fedc51bf23d328c Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 23 Aug 2022 10:15:11 +0900
Subject: [PATCH 0943/3199] save

---
 scheduler/src/lib.rs | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 823385bce2f358..778f25f989386c 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -466,6 +466,10 @@ impl Task {
     fn still_contended(&self) -> bool {
         !self.uncontended.load(std::sync::atomic::Ordering::SeqCst)
     }
+
+    fn mark_as_uncontended(&self) {
+        self.uncontended.store(true, std::sync::atomic::Ordering::SeqCst)
+    }
 }
 
 // RunnableQueue, ContendedQueue?
@@ -892,7 +896,7 @@ impl ScheduleStage {
         let mut rng = rand::thread_rng();
         // load account now from AccountsDb
         let checkpointed_contended_queue = contended_queue.clone();
-        task.uncontended.store(true, std::sync::atomic::Ordering::SeqCst);
+        task.mark_as_uncontended();
 
         Box::new(ExecutionEnvironment {
             task,

From c83a070611e28323fd2829d7eb5562d5ed0c2989 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 23 Aug 2022 10:15:58 +0900
Subject: [PATCH 0944/3199] save

---
 scheduler/src/lib.rs | 10 +++++++---
 1 file changed, 7 insertions(+), 3 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 778f25f989386c..eb2a714a997605 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -459,16 +459,20 @@ struct Bundle {
 pub struct Task {
     pub tx: Box<(SanitizedTransaction, Vec<LockAttempt>)>, // actually should be Bundle
     pub contention_count: usize,
-    pub uncontended: std::sync::atomic::AtomicBool,
+    pub uncontended: std::sync::atomic::AtomicUsize,
 }
 
 impl Task {
     fn still_contended(&self) -> bool {
-        !self.uncontended.load(std::sync::atomic::Ordering::SeqCst)
+        self.uncontended.load(std::sync::atomic::Ordering::SeqCst) < 2
+    }
+
+    fn mark_as_contended(&self) {
+        self.uncontended.store(1, std::sync::atomic::Ordering::SeqCst)
     }
 
     fn mark_as_uncontended(&self) {
-        self.uncontended.store(true, std::sync::atomic::Ordering::SeqCst)
+        self.uncontended.store(2, std::sync::atomic::Ordering::SeqCst)
     }
 }
 

From 793550317cca6b7284914c701f198ddd4e7eb982 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 23 Aug 2022 10:17:08 +0900
Subject: [PATCH 0945/3199] save

---
 scheduler/src/lib.rs | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index eb2a714a997605..6fe5dd61b604fb 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -463,8 +463,8 @@ pub struct Task {
 }
 
 impl Task {
-    fn still_contended(&self) -> bool {
-        self.uncontended.load(std::sync::atomic::Ordering::SeqCst) < 2
+    fn currently_contended(&self) -> bool {
+        self.uncontended.load(std::sync::atomic::Ordering::SeqCst) == 1
     }
 
     fn mark_as_contended(&self) {
@@ -514,7 +514,7 @@ impl TaskQueue {
     pub fn has_task(&self, unique_weight: &UniqueWeight) -> bool {
         let maybe_task = self.tasks.get(unique_weight);
         match maybe_task {
-            Some(task) => task.still_contended(),
+            Some(task) => task.currently_contended(),
             None => false,
         }
     }

From d5e31b2f342d2f291f3cc542af85976f839bc73c Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 23 Aug 2022 10:25:30 +0900
Subject: [PATCH 0946/3199] save

---
 scheduler/src/lib.rs | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 6fe5dd61b604fb..1e2075aea586bd 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -587,7 +587,7 @@ pub fn get_transaction_priority_details(tx: &SanitizedTransaction) -> u64 {
 pub struct ScheduleStage {}
 
 impl ScheduleStage {
-    fn push_to_queue(
+    fn push_to_runnable_queue(
         (weight, tx): (Weight, Box<(SanitizedTransaction, Vec<LockAttempt>)>),
         runnable_queue: &mut TaskQueue,
         unique_key: &mut u64,
@@ -746,9 +746,11 @@ impl ScheduleStage {
 
                 if from_runnable {
                     trace!("move to contended due to lock failure [{}/{}/{}]", unlockable_count, provisional_count, lock_count);
+                    let task = queue_entry.remove();
+                    task.mark_as_contended();
                     reborrowed_contended_queue
                         .unwrap()
-                        .add_to_schedule(*queue_entry.key(), queue_entry.remove());
+                        .add_to_schedule(*queue_entry.key(), task);
                     // maybe run lightweight prune logic on contended_queue here.
                 } else {
                     trace!("relock failed [{}/{}/{}]; remains in contended: {:?} contention: {}", unlockable_count, provisional_count, lock_count, &unique_weight, next_task.contention_count);
@@ -943,7 +945,7 @@ impl ScheduleStage {
         runnable_queue: &mut TaskQueue,
         unique_key: &mut u64,
     ) {
-        Self::push_to_queue(weighted_tx, runnable_queue, unique_key)
+        Self::push_to_runnable_queue(weighted_tx, runnable_queue, unique_key)
     }
 
     pub fn run(

From 755ff8934bbaaa683ca2c6bc059ece9775860261 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 23 Aug 2022 10:26:10 +0900
Subject: [PATCH 0947/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 1e2075aea586bd..28811536ff4d07 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -750,7 +750,7 @@ impl ScheduleStage {
                     task.mark_as_contended();
                     reborrowed_contended_queue
                         .unwrap()
-                        .add_to_schedule(*queue_entry.key(), task);
+                        .add_to_schedule(unique_weight, task);
                     // maybe run lightweight prune logic on contended_queue here.
                 } else {
                     trace!("relock failed [{}/{}/{}]; remains in contended: {:?} contention: {}", unlockable_count, provisional_count, lock_count, &unique_weight, next_task.contention_count);

From b61b19479ed1a29db212869e7d057856da6d20a2 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 23 Aug 2022 10:31:27 +0900
Subject: [PATCH 0948/3199] save

---
 scheduler/src/lib.rs | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 28811536ff4d07..8d43db6c521632 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -480,8 +480,9 @@ impl Task {
 #[derive(Default, Debug, Clone)]
 pub struct TaskQueue {
     //tasks: std::collections::BTreeMap<UniqueWeight, Task>,
-    tasks: im::OrdMap<UniqueWeight, TaskInQueue>,
+    //tasks: im::OrdMap<UniqueWeight, TaskInQueue>,
     //tasks: im::HashMap<UniqueWeight, TaskInQueue>,
+    tasks: Arc<dashmap::DashMap<UniqueWeight, TaskInQueue>>,
 }
 
 type TaskInQueue = std::sync::Arc<Task>;

From 5ac8b57ac14e5ec8ff9e0ee5dff1df7261ffc3fd Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 23 Aug 2022 10:31:51 +0900
Subject: [PATCH 0949/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 8d43db6c521632..bbabd99ebbb0a8 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -482,7 +482,7 @@ pub struct TaskQueue {
     //tasks: std::collections::BTreeMap<UniqueWeight, Task>,
     //tasks: im::OrdMap<UniqueWeight, TaskInQueue>,
     //tasks: im::HashMap<UniqueWeight, TaskInQueue>,
-    tasks: Arc<dashmap::DashMap<UniqueWeight, TaskInQueue>>,
+    tasks: std::sync::Arc<dashmap::DashMap<UniqueWeight, TaskInQueue>>,
 }
 
 type TaskInQueue = std::sync::Arc<Task>;

From 5c48ce956dccb3992819785d6761e96605264bde Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 23 Aug 2022 10:32:42 +0900
Subject: [PATCH 0950/3199] save

---
 scheduler/src/lib.rs | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index bbabd99ebbb0a8..00ccfe515f3d1e 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -486,10 +486,12 @@ pub struct TaskQueue {
 }
 
 type TaskInQueue = std::sync::Arc<Task>;
-type TaskQueueEntry<'a> = im::ordmap::Entry<'a, UniqueWeight, TaskInQueue>;
-type TaskQueueOccupiedEntry<'a> = im::ordmap::OccupiedEntry<'a, UniqueWeight, TaskInQueue>;
+//type TaskQueueEntry<'a> = im::ordmap::Entry<'a, UniqueWeight, TaskInQueue>;
+//type TaskQueueOccupiedEntry<'a> = im::ordmap::OccupiedEntry<'a, UniqueWeight, TaskInQueue>;
 //type TaskQueueEntry<'a> = im::hashmap::Entry<'a, UniqueWeight, TaskInQueue, , std::collections::hash_map::RandomState>;
 //type TaskQueueOccupiedEntry<'a> = im::hashmap::OccupiedEntry<'a, UniqueWeight, TaskInQueue, std::collections::hash_map::RandomState>;
+type TaskQueueEntry<'a> = dashmap::mapref::entry::Entry<'a, UniqueWeight, TaskInQueue>;
+type TaskQueueOccupiedEntry<'a> = dashmap::mapref::entry::OccupiedEntry<'a, UniqueWeight, TaskInQueue>;
 
 impl TaskQueue {
     #[inline(never)]

From 14169515d7592fdb93b008dca532f84cbf7adbf4 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 23 Aug 2022 10:33:08 +0900
Subject: [PATCH 0951/3199] save

---
 scheduler/src/lib.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 00ccfe515f3d1e..01a24016841c9b 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -488,10 +488,10 @@ pub struct TaskQueue {
 type TaskInQueue = std::sync::Arc<Task>;
 //type TaskQueueEntry<'a> = im::ordmap::Entry<'a, UniqueWeight, TaskInQueue>;
 //type TaskQueueOccupiedEntry<'a> = im::ordmap::OccupiedEntry<'a, UniqueWeight, TaskInQueue>;
-//type TaskQueueEntry<'a> = im::hashmap::Entry<'a, UniqueWeight, TaskInQueue, , std::collections::hash_map::RandomState>;
+//type TaskQueueEntry<'a> = im::hashmap::Entry<'a, UniqueWeight, TaskInQueue, std::collections::hash_map::RandomState>;
 //type TaskQueueOccupiedEntry<'a> = im::hashmap::OccupiedEntry<'a, UniqueWeight, TaskInQueue, std::collections::hash_map::RandomState>;
 type TaskQueueEntry<'a> = dashmap::mapref::entry::Entry<'a, UniqueWeight, TaskInQueue>;
-type TaskQueueOccupiedEntry<'a> = dashmap::mapref::entry::OccupiedEntry<'a, UniqueWeight, TaskInQueue>;
+type TaskQueueOccupiedEntry<'a> = dashmap::mapref::entry::OccupiedEntry<'a, UniqueWeight, TaskInQueue, std::collections::hash_map::RandomState>;
 
 impl TaskQueue {
     #[inline(never)]

From e23274ea69651ad083be3a361b6196db3cf24906 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 23 Aug 2022 10:33:20 +0900
Subject: [PATCH 0952/3199] save

---
 scheduler/src/lib.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 01a24016841c9b..510d8cce728171 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -527,8 +527,8 @@ impl TaskQueue {
         &mut self,
     ) -> Option<TaskQueueOccupiedEntry<'_>> {
         //panic!()//self.tasks.last_entry()
-        let k = self.tasks.get_max().map(|(k, _v)| *k);
-        //let k = self.tasks.iter().next().map(|(k, _v)| *k);
+        //let k = self.tasks.get_max().map(|(k, _v)| *k);
+        let k = self.tasks.iter().next().map(|(k, _v)| *k);
         k.map(|k| self.entry_to_execute(k).unwrap())
     }
 

From 0a30a6b05e90e985b1c5fd7e7d43e6c990de2e73 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 23 Aug 2022 10:33:57 +0900
Subject: [PATCH 0953/3199] save

---
 scheduler/src/lib.rs | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 510d8cce728171..c8003f80710132 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -528,8 +528,9 @@ impl TaskQueue {
     ) -> Option<TaskQueueOccupiedEntry<'_>> {
         //panic!()//self.tasks.last_entry()
         //let k = self.tasks.get_max().map(|(k, _v)| *k);
-        let k = self.tasks.iter().next().map(|(k, _v)| *k);
-        k.map(|k| self.entry_to_execute(k).unwrap())
+        panic!();
+        //let k = self.tasks.iter().next().map(|(k, _v)| *k);
+        //k.map(|k| self.entry_to_execute(k).unwrap())
     }
 
     fn task_count(&self) -> usize {

From d578c1a67fd26af2e2a5c9a6145025ed2f3dc40e Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 23 Aug 2022 10:36:31 +0900
Subject: [PATCH 0954/3199] save

---
 scheduler/src/lib.rs | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index c8003f80710132..df5bdb9ffda8e2 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -528,9 +528,9 @@ impl TaskQueue {
     ) -> Option<TaskQueueOccupiedEntry<'_>> {
         //panic!()//self.tasks.last_entry()
         //let k = self.tasks.get_max().map(|(k, _v)| *k);
-        panic!();
-        //let k = self.tasks.iter().next().map(|(k, _v)| *k);
-        //k.map(|k| self.entry_to_execute(k).unwrap())
+        //panic!();
+        let k = self.tasks.iter().next().map(|(k, _v)| *k);
+        k.map(|k| self.entry_to_execute(k).unwrap())
     }
 
     fn task_count(&self) -> usize {

From 8f37730bc3ba127112572225085bfcd8ac4f3fa0 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 23 Aug 2022 10:37:55 +0900
Subject: [PATCH 0955/3199] save

---
 scheduler/src/lib.rs | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index df5bdb9ffda8e2..429ee3aaae27b7 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -529,7 +529,8 @@ impl TaskQueue {
         //panic!()//self.tasks.last_entry()
         //let k = self.tasks.get_max().map(|(k, _v)| *k);
         //panic!();
-        let k = self.tasks.iter().next().map(|(k, _v)| *k);
+        //let k = self.tasks.iter().next().map(|(k, _v)| *k);
+        let k = self.tasks.iter().next().map(|r| *r.key());
         k.map(|k| self.entry_to_execute(k).unwrap())
     }
 

From c83d9c0d1866da2da3d7c4502328a5ee7773805b Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 23 Aug 2022 10:41:53 +0900
Subject: [PATCH 0956/3199] save

---
 scheduler/src/lib.rs | 21 +++++++++++----------
 1 file changed, 11 insertions(+), 10 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 429ee3aaae27b7..5492001099bf0e 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -480,18 +480,18 @@ impl Task {
 #[derive(Default, Debug, Clone)]
 pub struct TaskQueue {
     //tasks: std::collections::BTreeMap<UniqueWeight, Task>,
-    //tasks: im::OrdMap<UniqueWeight, TaskInQueue>,
+    tasks: im::OrdMap<UniqueWeight, TaskInQueue>,
     //tasks: im::HashMap<UniqueWeight, TaskInQueue>,
-    tasks: std::sync::Arc<dashmap::DashMap<UniqueWeight, TaskInQueue>>,
+    //tasks: std::sync::Arc<dashmap::DashMap<UniqueWeight, TaskInQueue>>,
 }
 
 type TaskInQueue = std::sync::Arc<Task>;
-//type TaskQueueEntry<'a> = im::ordmap::Entry<'a, UniqueWeight, TaskInQueue>;
-//type TaskQueueOccupiedEntry<'a> = im::ordmap::OccupiedEntry<'a, UniqueWeight, TaskInQueue>;
+type TaskQueueEntry<'a> = im::ordmap::Entry<'a, UniqueWeight, TaskInQueue>;
+type TaskQueueOccupiedEntry<'a> = im::ordmap::OccupiedEntry<'a, UniqueWeight, TaskInQueue>;
 //type TaskQueueEntry<'a> = im::hashmap::Entry<'a, UniqueWeight, TaskInQueue, std::collections::hash_map::RandomState>;
 //type TaskQueueOccupiedEntry<'a> = im::hashmap::OccupiedEntry<'a, UniqueWeight, TaskInQueue, std::collections::hash_map::RandomState>;
-type TaskQueueEntry<'a> = dashmap::mapref::entry::Entry<'a, UniqueWeight, TaskInQueue>;
-type TaskQueueOccupiedEntry<'a> = dashmap::mapref::entry::OccupiedEntry<'a, UniqueWeight, TaskInQueue, std::collections::hash_map::RandomState>;
+//type TaskQueueEntry<'a> = dashmap::mapref::entry::Entry<'a, UniqueWeight, TaskInQueue>;
+//type TaskQueueOccupiedEntry<'a> = dashmap::mapref::entry::OccupiedEntry<'a, UniqueWeight, TaskInQueue, std::collections::hash_map::RandomState>;
 
 impl TaskQueue {
     #[inline(never)]
@@ -514,7 +514,7 @@ impl TaskQueue {
     }
 
     #[inline(never)]
-    pub fn has_task(&self, unique_weight: &UniqueWeight) -> bool {
+    pub fn has_contended_task(&self, unique_weight: &UniqueWeight) -> bool {
         let maybe_task = self.tasks.get(unique_weight);
         match maybe_task {
             Some(task) => task.currently_contended(),
@@ -527,10 +527,11 @@ impl TaskQueue {
         &mut self,
     ) -> Option<TaskQueueOccupiedEntry<'_>> {
         //panic!()//self.tasks.last_entry()
-        //let k = self.tasks.get_max().map(|(k, _v)| *k);
+        let k = self.tasks.get_max().map(|(k, _v)| *k);
         //panic!();
         //let k = self.tasks.iter().next().map(|(k, _v)| *k);
-        let k = self.tasks.iter().next().map(|r| *r.key());
+        //let k = self.tasks.iter().next().map(|r| *r.key());
+
         k.map(|k| self.entry_to_execute(k).unwrap())
     }
 
@@ -866,7 +867,7 @@ impl ScheduleStage {
             if newly_uncontended_while_queued && page.next_usage == Usage::Unused {
                 let task_id = l.heaviest_uncontended.load(std::sync::atomic::Ordering::SeqCst);
                 if task_id != 0 {
-                    if contended_queue.has_task(&task_id) {
+                    if contended_queue.has_contended_task(&task_id) {
                         address_book.uncontended_task_ids.insert(task_id, ());
                     }
                 }

From ef02d8c9e1bdbd03282d54b60d12a3feba77ff2b Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 23 Aug 2022 10:42:13 +0900
Subject: [PATCH 0957/3199] save

---
 transaction-scheduler-bench/src/main.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/transaction-scheduler-bench/src/main.rs b/transaction-scheduler-bench/src/main.rs
index 8cd003d5ccfd0e..d3b12d12b9e67f 100644
--- a/transaction-scheduler-bench/src/main.rs
+++ b/transaction-scheduler-bench/src/main.rs
@@ -350,7 +350,7 @@ fn handle_transaction_batch(
         page.contended_unique_weights.remove_task_id(&uq);
         if let Some(mut task_cursor) = page.contended_unique_weights.heaviest_task_cursor() {
             let mut found = true;
-            while !transaction_batch.checkpointed_contended_queue.has_task(task_cursor.value()) {
+            while !transaction_batch.checkpointed_contended_queue.has_contended_task(task_cursor.value()) {
                 if let Some(new_cursor) = task_cursor.prev() {
                     task_cursor = new_cursor;
                 } else {

From 3612e36ab46bf9acf0d133d911d97b05729fa323 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 23 Aug 2022 10:45:23 +0900
Subject: [PATCH 0958/3199] save

---
 scheduler/src/lib.rs | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 5492001099bf0e..21ca782ee4bcc6 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -480,18 +480,18 @@ impl Task {
 #[derive(Default, Debug, Clone)]
 pub struct TaskQueue {
     //tasks: std::collections::BTreeMap<UniqueWeight, Task>,
-    tasks: im::OrdMap<UniqueWeight, TaskInQueue>,
+    //tasks: im::OrdMap<UniqueWeight, TaskInQueue>,
     //tasks: im::HashMap<UniqueWeight, TaskInQueue>,
-    //tasks: std::sync::Arc<dashmap::DashMap<UniqueWeight, TaskInQueue>>,
+    tasks: std::sync::Arc<dashmap::DashMap<UniqueWeight, TaskInQueue>>,
 }
 
 type TaskInQueue = std::sync::Arc<Task>;
-type TaskQueueEntry<'a> = im::ordmap::Entry<'a, UniqueWeight, TaskInQueue>;
-type TaskQueueOccupiedEntry<'a> = im::ordmap::OccupiedEntry<'a, UniqueWeight, TaskInQueue>;
+//type TaskQueueEntry<'a> = im::ordmap::Entry<'a, UniqueWeight, TaskInQueue>;
+//type TaskQueueOccupiedEntry<'a> = im::ordmap::OccupiedEntry<'a, UniqueWeight, TaskInQueue>;
 //type TaskQueueEntry<'a> = im::hashmap::Entry<'a, UniqueWeight, TaskInQueue, std::collections::hash_map::RandomState>;
 //type TaskQueueOccupiedEntry<'a> = im::hashmap::OccupiedEntry<'a, UniqueWeight, TaskInQueue, std::collections::hash_map::RandomState>;
-//type TaskQueueEntry<'a> = dashmap::mapref::entry::Entry<'a, UniqueWeight, TaskInQueue>;
-//type TaskQueueOccupiedEntry<'a> = dashmap::mapref::entry::OccupiedEntry<'a, UniqueWeight, TaskInQueue, std::collections::hash_map::RandomState>;
+type TaskQueueEntry<'a> = dashmap::mapref::entry::Entry<'a, UniqueWeight, TaskInQueue>;
+type TaskQueueOccupiedEntry<'a> = dashmap::mapref::entry::OccupiedEntry<'a, UniqueWeight, TaskInQueue, std::collections::hash_map::RandomState>;
 
 impl TaskQueue {
     #[inline(never)]

From 50865f85e4b50d5748633cd4051f46d081437848 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 23 Aug 2022 10:45:36 +0900
Subject: [PATCH 0959/3199] save

---
 scheduler/src/lib.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 21ca782ee4bcc6..56029d949007d4 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -527,10 +527,10 @@ impl TaskQueue {
         &mut self,
     ) -> Option<TaskQueueOccupiedEntry<'_>> {
         //panic!()//self.tasks.last_entry()
-        let k = self.tasks.get_max().map(|(k, _v)| *k);
+        //let k = self.tasks.get_max().map(|(k, _v)| *k);
         //panic!();
         //let k = self.tasks.iter().next().map(|(k, _v)| *k);
-        //let k = self.tasks.iter().next().map(|r| *r.key());
+        let k = self.tasks.iter().next().map(|r| *r.key());
 
         k.map(|k| self.entry_to_execute(k).unwrap())
     }

From 8f447cc98267fa55f8b81e16796e9cb627e97a02 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 23 Aug 2022 10:49:49 +0900
Subject: [PATCH 0960/3199] save

---
 scheduler/src/lib.rs                    | 2 +-
 transaction-scheduler-bench/src/main.rs | 3 ++-
 2 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 56029d949007d4..fe90af71acdba7 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -574,7 +574,7 @@ type PreprocessedTransaction = (SanitizedTransaction, Vec<LockAttempt>);
 // adaptive relative load between sigverify stage and execution substage
 // switched from crossbeam_channel::select! due to observed poor performance
 pub enum Multiplexed {
-    FromPrevious((Weight, Box<PreprocessedTransaction>)),
+    FromPrevious((Weight, std::sync::Arc<PreprocessedTransaction>)),
     FromPreviousBatched(Vec<Vec<Box<PreprocessedTransaction>>>),
 }
 
diff --git a/transaction-scheduler-bench/src/main.rs b/transaction-scheduler-bench/src/main.rs
index d3b12d12b9e67f..f69485220a4c32 100644
--- a/transaction-scheduler-bench/src/main.rs
+++ b/transaction-scheduler-bench/src/main.rs
@@ -471,8 +471,9 @@ fn send_packets(
             for v in vv {
                 let p = solana_scheduler::get_transaction_priority_details(&v.0);
                 let p = (p << 32) | (rng.gen::<u64>() & 0x0000_0000_ffff_ffff);
+                let v = std::sync::Arc::new(v);
                 for lock_attempt in v.1.iter() {
-                    lock_attempt.target.page_ref().contended_unique_weights.insert_task_id(p);
+                    lock_attempt.target.page_ref().contended_unique_weights.insert_task_id(p, v);
                 }
                 packet_batch_sender.send(solana_scheduler::Multiplexed::FromPrevious((p, v))).unwrap();
             }

From b038a90e8ac13676d22e2dd5957e369d9cfe05d0 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 23 Aug 2022 10:51:15 +0900
Subject: [PATCH 0961/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index fe90af71acdba7..602732f4d2edfa 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -947,7 +947,7 @@ impl ScheduleStage {
 
     #[inline(never)]
     fn register_runnable_task(
-        weighted_tx: (Weight, Box<(SanitizedTransaction, Vec<LockAttempt>)>),
+        weighted_tx: (Weight, std::sync::Arc<(SanitizedTransaction, Vec<LockAttempt>)>),
         runnable_queue: &mut TaskQueue,
         unique_key: &mut u64,
     ) {

From 369a283442b96021bb43981bdbc0bd8975187265 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 23 Aug 2022 10:51:29 +0900
Subject: [PATCH 0962/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 602732f4d2edfa..eba34f947e2d7e 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -594,7 +594,7 @@ pub struct ScheduleStage {}
 
 impl ScheduleStage {
     fn push_to_runnable_queue(
-        (weight, tx): (Weight, Box<(SanitizedTransaction, Vec<LockAttempt>)>),
+        (weight, tx): (Weight, std::sync::Arc<(SanitizedTransaction, Vec<LockAttempt>)>),
         runnable_queue: &mut TaskQueue,
         unique_key: &mut u64,
     ) {

From 6c97c8bc88b3c0c3b70dde2f426d8572b50982fb Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 23 Aug 2022 10:53:36 +0900
Subject: [PATCH 0963/3199] save

---
 transaction-scheduler-bench/src/main.rs | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/transaction-scheduler-bench/src/main.rs b/transaction-scheduler-bench/src/main.rs
index f69485220a4c32..e0dd3945d57ba8 100644
--- a/transaction-scheduler-bench/src/main.rs
+++ b/transaction-scheduler-bench/src/main.rs
@@ -471,11 +471,11 @@ fn send_packets(
             for v in vv {
                 let p = solana_scheduler::get_transaction_priority_details(&v.0);
                 let p = (p << 32) | (rng.gen::<u64>() & 0x0000_0000_ffff_ffff);
-                let v = std::sync::Arc::new(v);
+                let t = solana_scheduler::Task::new_arced(v);
                 for lock_attempt in v.1.iter() {
-                    lock_attempt.target.page_ref().contended_unique_weights.insert_task_id(p, v);
+                    lock_attempt.target.page_ref().contended_unique_weights.insert_task_id(p, t);
                 }
-                packet_batch_sender.send(solana_scheduler::Multiplexed::FromPrevious((p, v))).unwrap();
+                packet_batch_sender.send(solana_scheduler::Multiplexed::FromPrevious((p, t))).unwrap();
             }
         }
         

From d2a4175422dacdd008124341341cf94413f0db90 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 23 Aug 2022 10:54:52 +0900
Subject: [PATCH 0964/3199] save

---
 scheduler/src/lib.rs | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index eba34f947e2d7e..bab9c13fb5273c 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -463,6 +463,10 @@ pub struct Task {
 }
 
 impl Task {
+    fn new_for_queue(v: usize) -> std::sync::Arc<Self> {
+        TaskInQueue::new(v)
+    }
+
     fn currently_contended(&self) -> bool {
         self.uncontended.load(std::sync::atomic::Ordering::SeqCst) == 1
     }

From 8d33eb5afbc9a22fee7dee73e81b18d76eda3427 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 23 Aug 2022 10:55:12 +0900
Subject: [PATCH 0965/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index bab9c13fb5273c..27a74d547c72c3 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -578,7 +578,7 @@ type PreprocessedTransaction = (SanitizedTransaction, Vec<LockAttempt>);
 // adaptive relative load between sigverify stage and execution substage
 // switched from crossbeam_channel::select! due to observed poor performance
 pub enum Multiplexed {
-    FromPrevious((Weight, std::sync::Arc<PreprocessedTransaction>)),
+    FromPrevious((Weight, TaskInQueue)),
     FromPreviousBatched(Vec<Vec<Box<PreprocessedTransaction>>>),
 }
 

From 61e42766a14d53a7c93f41372cfb9365bf0accdb Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 23 Aug 2022 10:56:16 +0900
Subject: [PATCH 0966/3199] save

---
 scheduler/src/lib.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 27a74d547c72c3..ebd20c8667265b 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -463,8 +463,8 @@ pub struct Task {
 }
 
 impl Task {
-    fn new_for_queue(v: usize) -> std::sync::Arc<Self> {
-        TaskInQueue::new(v)
+    fn new_for_queue(v: Box<(SanitizedTransaction, Vec<LockAttempt>)>) -> std::sync::Arc<Self> {
+        TaskInQueue::new(Task { tx, contention_count: 0, uncontended: Default::default() })
     }
 
     fn currently_contended(&self) -> bool {

From 34de69d21483028bd99af7277e9e8557d0d042d3 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 23 Aug 2022 10:56:37 +0900
Subject: [PATCH 0967/3199] save

---
 scheduler/src/lib.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index ebd20c8667265b..d8050cd2da17be 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -463,8 +463,8 @@ pub struct Task {
 }
 
 impl Task {
-    fn new_for_queue(v: Box<(SanitizedTransaction, Vec<LockAttempt>)>) -> std::sync::Arc<Self> {
-        TaskInQueue::new(Task { tx, contention_count: 0, uncontended: Default::default() })
+    fn new_for_queue(tx: Box<(SanitizedTransaction, Vec<LockAttempt>)>) -> std::sync::Arc<Self> {
+        TaskInQueue::new(Self { tx, contention_count: 0, uncontended: Default::default() })
     }
 
     fn currently_contended(&self) -> bool {

From 152a3b20a4b5619a2ad6a74a8ecb7263de26eb91 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 23 Aug 2022 10:57:33 +0900
Subject: [PATCH 0968/3199] save

---
 scheduler/src/lib.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index d8050cd2da17be..02b45d5da89ee2 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -598,7 +598,7 @@ pub struct ScheduleStage {}
 
 impl ScheduleStage {
     fn push_to_runnable_queue(
-        (weight, tx): (Weight, std::sync::Arc<(SanitizedTransaction, Vec<LockAttempt>)>),
+        task: TaskInQueue,
         runnable_queue: &mut TaskQueue,
         unique_key: &mut u64,
     ) {
@@ -625,7 +625,7 @@ impl ScheduleStage {
             },
             */
             unique_weight,
-            TaskInQueue::new(Task { tx, contention_count: 0, uncontended: Default::default() }),
+            task,
         );
         *unique_key -= 1;
     }

From b0d353c8bae88af002a4bf85f2ad775e896612dd Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 23 Aug 2022 10:58:01 +0900
Subject: [PATCH 0969/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 02b45d5da89ee2..2dc63db9ded3a4 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -598,7 +598,7 @@ pub struct ScheduleStage {}
 
 impl ScheduleStage {
     fn push_to_runnable_queue(
-        task: TaskInQueue,
+        task: (Weight, TaskInQueue),
         runnable_queue: &mut TaskQueue,
         unique_key: &mut u64,
     ) {

From 23c644bb3d5ac5b5c7d6862a693792dd3e410bc8 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 23 Aug 2022 10:58:16 +0900
Subject: [PATCH 0970/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 2dc63db9ded3a4..6278c13559c899 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -598,7 +598,7 @@ pub struct ScheduleStage {}
 
 impl ScheduleStage {
     fn push_to_runnable_queue(
-        task: (Weight, TaskInQueue),
+        (weight, tx): (Weight, TaskInQueue),
         runnable_queue: &mut TaskQueue,
         unique_key: &mut u64,
     ) {

From ca461bf93cccb33bcb43c85d4e4089ddb9eca26e Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 23 Aug 2022 10:58:33 +0900
Subject: [PATCH 0971/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 6278c13559c899..c893dcc9b2b1a2 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -951,7 +951,7 @@ impl ScheduleStage {
 
     #[inline(never)]
     fn register_runnable_task(
-        weighted_tx: (Weight, std::sync::Arc<(SanitizedTransaction, Vec<LockAttempt>)>),
+        weighted_tx: (Weight, TaskInQueue)>),
         runnable_queue: &mut TaskQueue,
         unique_key: &mut u64,
     ) {

From db5fde41920d80fd3d827ddd2d65e50c433da534 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 23 Aug 2022 10:58:41 +0900
Subject: [PATCH 0972/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index c893dcc9b2b1a2..a1a02b14b9f25e 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -951,7 +951,7 @@ impl ScheduleStage {
 
     #[inline(never)]
     fn register_runnable_task(
-        weighted_tx: (Weight, TaskInQueue)>),
+        weighted_tx: (Weight, TaskInQueue),
         runnable_queue: &mut TaskQueue,
         unique_key: &mut u64,
     ) {

From ff3f55efb66caf5de3f8f4cff86cd290dbce2098 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 23 Aug 2022 10:59:01 +0900
Subject: [PATCH 0973/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index a1a02b14b9f25e..e82b4170bf1246 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -598,7 +598,7 @@ pub struct ScheduleStage {}
 
 impl ScheduleStage {
     fn push_to_runnable_queue(
-        (weight, tx): (Weight, TaskInQueue),
+        (weight, task): (Weight, TaskInQueue),
         runnable_queue: &mut TaskQueue,
         unique_key: &mut u64,
     ) {

From 1163ec63d45d49dc131cabbc901a251f88d9b8dc Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 23 Aug 2022 11:01:30 +0900
Subject: [PATCH 0974/3199] save

---
 transaction-scheduler-bench/src/main.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/transaction-scheduler-bench/src/main.rs b/transaction-scheduler-bench/src/main.rs
index e0dd3945d57ba8..a5385cca36d2a6 100644
--- a/transaction-scheduler-bench/src/main.rs
+++ b/transaction-scheduler-bench/src/main.rs
@@ -471,7 +471,7 @@ fn send_packets(
             for v in vv {
                 let p = solana_scheduler::get_transaction_priority_details(&v.0);
                 let p = (p << 32) | (rng.gen::<u64>() & 0x0000_0000_ffff_ffff);
-                let t = solana_scheduler::Task::new_arced(v);
+                let t = solana_scheduler::Task::new_for_queue(v);
                 for lock_attempt in v.1.iter() {
                     lock_attempt.target.page_ref().contended_unique_weights.insert_task_id(p, t);
                 }

From 3f003eda341253a66a832b354225bdd585073026 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 23 Aug 2022 11:01:40 +0900
Subject: [PATCH 0975/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index e82b4170bf1246..18830b2d02fdfb 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -463,7 +463,7 @@ pub struct Task {
 }
 
 impl Task {
-    fn new_for_queue(tx: Box<(SanitizedTransaction, Vec<LockAttempt>)>) -> std::sync::Arc<Self> {
+    pub fn new_for_queue(tx: Box<(SanitizedTransaction, Vec<LockAttempt>)>) -> std::sync::Arc<Self> {
         TaskInQueue::new(Self { tx, contention_count: 0, uncontended: Default::default() })
     }
 

From 63bc3ac2fa5754d9fb4a8775ba0da7f43b9c1fa0 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 23 Aug 2022 11:02:20 +0900
Subject: [PATCH 0976/3199] save

---
 scheduler/src/lib.rs | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 18830b2d02fdfb..3b29ca2d01cf82 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -122,13 +122,13 @@ pub enum RequestedUsage {
 #[derive(Debug, Default)]
 pub struct TaskIds {
     //task_ids: std::collections::BTreeSet<UniqueWeight>,
-    task_ids: crossbeam_skiplist::SkipSet<UniqueWeight>,
+    task_ids: crossbeam_skiplist::SkipSet<UniqueWeight, TaskInQueue>,
     //cached_heaviest: Option<UniqueWeight>,
 }
 
 impl TaskIds {
     #[inline(never)]
-    pub fn insert_task_id(&self, u: TaskId) {
+    pub fn insert_task_id(&self, u: TaskId, task: TaskInQueue) {
         /*
         match self.cached_heaviest {
             Some(c) if u > c => { self.cached_heaviest = Some(u) },
@@ -137,7 +137,7 @@ impl TaskIds {
         }
         */
              
-        self.task_ids.insert(u);
+        self.task_ids.insert(u, task);
     }
 
     #[inline(never)]

From dccf1565f7d985646af89baf225ac473a2ee9de5 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 23 Aug 2022 11:02:29 +0900
Subject: [PATCH 0977/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 3b29ca2d01cf82..31f527aeecf16e 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -122,7 +122,7 @@ pub enum RequestedUsage {
 #[derive(Debug, Default)]
 pub struct TaskIds {
     //task_ids: std::collections::BTreeSet<UniqueWeight>,
-    task_ids: crossbeam_skiplist::SkipSet<UniqueWeight, TaskInQueue>,
+    task_ids: crossbeam_skiplist::SkipMap<UniqueWeight, TaskInQueue>,
     //cached_heaviest: Option<UniqueWeight>,
 }
 

From e66b053b5308646277021132db8a75cf3318dc7d Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 23 Aug 2022 11:03:03 +0900
Subject: [PATCH 0978/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 31f527aeecf16e..cdfa8fad3e6157 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -161,7 +161,7 @@ impl TaskIds {
     #[inline(never)]
     fn heaviest_task_id(&self) -> Option<TaskId> {
         //self.task_ids.last()
-        self.task_ids.back().map(|e| *(e.value()))
+        self.task_ids.back().map(|e| *(e.key()))
         //self.cached_heaviest.as_ref()
     }
 

From 15cf40a5c2f7a6998bd6dce580d7cb3b2f328bc7 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 23 Aug 2022 11:03:38 +0900
Subject: [PATCH 0979/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index cdfa8fad3e6157..da5eb75ce9c389 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -166,7 +166,7 @@ impl TaskIds {
     }
 
     #[inline(never)]
-    pub fn heaviest_task_cursor(&self) -> Option<crossbeam_skiplist::set::Entry<'_, u64>> {
+    pub fn heaviest_task_cursor(&self) -> Option<crossbeam_skiplist::map::Entry<'_, UniqueWeight, TaskInQueue>> {
         //self.task_ids.last()
         self.task_ids.back()
         //self.cached_heaviest.as_ref()

From 72597d4c43233e7662dfeeb00932111df3cf7485 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 23 Aug 2022 11:04:45 +0900
Subject: [PATCH 0980/3199] save

---
 transaction-scheduler-bench/src/main.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/transaction-scheduler-bench/src/main.rs b/transaction-scheduler-bench/src/main.rs
index a5385cca36d2a6..ff0a5299f4d9a9 100644
--- a/transaction-scheduler-bench/src/main.rs
+++ b/transaction-scheduler-bench/src/main.rs
@@ -350,7 +350,7 @@ fn handle_transaction_batch(
         page.contended_unique_weights.remove_task_id(&uq);
         if let Some(mut task_cursor) = page.contended_unique_weights.heaviest_task_cursor() {
             let mut found = true;
-            while !transaction_batch.checkpointed_contended_queue.has_contended_task(task_cursor.value()) {
+            while !task_cursor.value().currently_contended() {
                 if let Some(new_cursor) = task_cursor.prev() {
                     task_cursor = new_cursor;
                 } else {

From dcc7b5d01ead5af998d1041b0b8f6ffc2a9de385 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 23 Aug 2022 11:04:56 +0900
Subject: [PATCH 0981/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index da5eb75ce9c389..c67493e992a2e8 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -467,7 +467,7 @@ impl Task {
         TaskInQueue::new(Self { tx, contention_count: 0, uncontended: Default::default() })
     }
 
-    fn currently_contended(&self) -> bool {
+    pub fn currently_contended(&self) -> bool {
         self.uncontended.load(std::sync::atomic::Ordering::SeqCst) == 1
     }
 

From b97b949c9a10791e1cdd83a069b5dbf5a52880f8 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 23 Aug 2022 11:12:54 +0900
Subject: [PATCH 0982/3199] save

---
 scheduler/Cargo.toml                    | 1 +
 scheduler/src/lib.rs                    | 2 +-
 transaction-scheduler-bench/src/main.rs | 2 +-
 3 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/scheduler/Cargo.toml b/scheduler/Cargo.toml
index 9db98da14f8e37..b678e0644e8ef6 100644
--- a/scheduler/Cargo.toml
+++ b/scheduler/Cargo.toml
@@ -12,6 +12,7 @@ publish = true
 
 [dependencies]
 atomic_enum = "0.1.1"
+arc-swap = "1.5.0"
 by_address = "1.0.4"
 crossbeam-channel = "0.5.5"
 crossbeam-skiplist = { path = "../../../crossbeam/crossbeam-skiplist" }
diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index c67493e992a2e8..49d562a3464d9e 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -72,7 +72,7 @@ pub struct LockAttempt {
     pub target: PageRc,
     status: LockStatus,
     requested_usage: RequestedUsage,
-    pub heaviest_uncontended: std::sync::Arc<std::sync::atomic::AtomicU64>,
+    pub heaviest_uncontended: ArcSwap<TaskInQueue>,
     //remembered: bool,
 }
 
diff --git a/transaction-scheduler-bench/src/main.rs b/transaction-scheduler-bench/src/main.rs
index ff0a5299f4d9a9..34a55dda95a6f1 100644
--- a/transaction-scheduler-bench/src/main.rs
+++ b/transaction-scheduler-bench/src/main.rs
@@ -359,7 +359,7 @@ fn handle_transaction_batch(
                 }
             }
             if found {
-                lock_attempt.heaviest_uncontended.store(*(task_cursor.value()), std::sync::atomic::Ordering::SeqCst);
+                lock_attempt.heaviest_uncontended.store(*(task_cursor.key()), std::sync::atomic::Ordering::SeqCst);
             }
         }
     }

From cf060dbfdbe5c86eeaa6a0e975d4523b429c2db9 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 23 Aug 2022 11:13:14 +0900
Subject: [PATCH 0983/3199] save

---
 Cargo.lock           | 1 +
 scheduler/src/lib.rs | 2 +-
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/Cargo.lock b/Cargo.lock
index e959c0290c219e..4592102ddee94a 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -6102,6 +6102,7 @@ dependencies = [
 name = "solana-scheduler"
 version = "1.11.5"
 dependencies = [
+ "arc-swap",
  "atomic_enum",
  "by_address",
  "crossbeam-channel",
diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 49d562a3464d9e..af1a0bf4a93c26 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -72,7 +72,7 @@ pub struct LockAttempt {
     pub target: PageRc,
     status: LockStatus,
     requested_usage: RequestedUsage,
-    pub heaviest_uncontended: ArcSwap<TaskInQueue>,
+    pub heaviest_uncontended: arc_swap::ArcSwap<TaskInQueue>,
     //remembered: bool,
 }
 

From f8561e2dc7b67670545a3b5111f57723fee5c47e Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 23 Aug 2022 11:13:31 +0900
Subject: [PATCH 0984/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index af1a0bf4a93c26..c4ae4ce3b6db6f 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -67,7 +67,7 @@ enum LockStatus {
     Failed,
 }
 
-#[derive(Clone, Debug)]
+#[derive(Debug)]
 pub struct LockAttempt {
     pub target: PageRc,
     status: LockStatus,

From b53737366f47392309461e6e561999d691d4cc59 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 23 Aug 2022 11:16:26 +0900
Subject: [PATCH 0985/3199] save

---
 scheduler/src/lib.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index c4ae4ce3b6db6f..3cd919d7d1a11f 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -67,12 +67,12 @@ enum LockStatus {
     Failed,
 }
 
-#[derive(Debug)]
+#[derive(Clone, Debug)]
 pub struct LockAttempt {
     pub target: PageRc,
     status: LockStatus,
     requested_usage: RequestedUsage,
-    pub heaviest_uncontended: arc_swap::ArcSwap<TaskInQueue>,
+    pub heaviest_uncontended: arc_swap::ArcSwapOption<TaskInQueue>,
     //remembered: bool,
 }
 

From 361ebf4a05ceaa32a79a2dce4bb48dd3921e8308 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 23 Aug 2022 11:16:42 +0900
Subject: [PATCH 0986/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 3cd919d7d1a11f..f2c59b17e03e86 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -67,7 +67,7 @@ enum LockStatus {
     Failed,
 }
 
-#[derive(Clone, Debug)]
+#[derive(Debug)]
 pub struct LockAttempt {
     pub target: PageRc,
     status: LockStatus,

From 55bbc49a7f7ec7e33c360e3406af2e956d0eed2e Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 23 Aug 2022 11:17:21 +0900
Subject: [PATCH 0987/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index f2c59b17e03e86..94628c0ec979fd 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -869,7 +869,7 @@ impl ScheduleStage {
 
             let page = l.target.page_mut();
             if newly_uncontended_while_queued && page.next_usage == Usage::Unused {
-                let task_id = l.heaviest_uncontended.load(std::sync::atomic::Ordering::SeqCst);
+                let task_id = l.heaviest_uncontended.load().load(std::sync::atomic::Ordering::SeqCst);
                 if task_id != 0 {
                     if contended_queue.has_contended_task(&task_id) {
                         address_book.uncontended_task_ids.insert(task_id, ());

From 2e954f81752834da444fc999488cb1da16b99f35 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 23 Aug 2022 11:17:32 +0900
Subject: [PATCH 0988/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 94628c0ec979fd..4ec0d994cf8a99 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -869,7 +869,7 @@ impl ScheduleStage {
 
             let page = l.target.page_mut();
             if newly_uncontended_while_queued && page.next_usage == Usage::Unused {
-                let task_id = l.heaviest_uncontended.load().load(std::sync::atomic::Ordering::SeqCst);
+                let task_id = l.heaviest_uncontended.load().unwrap().load(std::sync::atomic::Ordering::SeqCst);
                 if task_id != 0 {
                     if contended_queue.has_contended_task(&task_id) {
                         address_book.uncontended_task_ids.insert(task_id, ());

From ea258beb6165ef27a9229c9a382630e8e5af017b Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 23 Aug 2022 11:18:13 +0900
Subject: [PATCH 0989/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 4ec0d994cf8a99..d1ab31844732c3 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -869,7 +869,7 @@ impl ScheduleStage {
 
             let page = l.target.page_mut();
             if newly_uncontended_while_queued && page.next_usage == Usage::Unused {
-                let task_id = l.heaviest_uncontended.load().unwrap().load(std::sync::atomic::Ordering::SeqCst);
+                let task_id = l.heaviest_uncontended.load_full().unwrap().load(std::sync::atomic::Ordering::SeqCst);
                 if task_id != 0 {
                     if contended_queue.has_contended_task(&task_id) {
                         address_book.uncontended_task_ids.insert(task_id, ());

From bf3e1383bb2767326564e66b4f08ab8ce974fae0 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 23 Aug 2022 11:18:58 +0900
Subject: [PATCH 0990/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index d1ab31844732c3..1f622138a047ed 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -72,7 +72,7 @@ pub struct LockAttempt {
     pub target: PageRc,
     status: LockStatus,
     requested_usage: RequestedUsage,
-    pub heaviest_uncontended: arc_swap::ArcSwapOption<TaskInQueue>,
+    pub heaviest_uncontended: arc_swap::ArcSwapOption<Task>,
     //remembered: bool,
 }
 

From b83d275b3dcf943c628a66a3d4767a2f165e543e Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 23 Aug 2022 11:21:57 +0900
Subject: [PATCH 0991/3199] save

---
 scheduler/src/lib.rs | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 1f622138a047ed..b49504c0b12f08 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -203,7 +203,7 @@ impl Page {
 //type AddressMap = std::collections::HashMap<Pubkey, PageRc>;
 type AddressMap = std::sync::Arc<dashmap::DashMap<Pubkey, PageRc>>;
 type TaskId = UniqueWeight;
-type WeightedTaskIds = std::collections::BTreeMap<TaskId, ()>;
+type WeightedTaskIds = std::collections::BTreeMap<TaskId, TaskInQueue>;
 //type AddressMapEntry<'a, K, V> = std::collections::hash_map::Entry<'a, K, V>;
 type AddressMapEntry<'a> = dashmap::mapref::entry::Entry<'a, Pubkey, PageRc>;
 
@@ -869,10 +869,10 @@ impl ScheduleStage {
 
             let page = l.target.page_mut();
             if newly_uncontended_while_queued && page.next_usage == Usage::Unused {
-                let task_id = l.heaviest_uncontended.load_full().unwrap().load(std::sync::atomic::Ordering::SeqCst);
-                if task_id != 0 {
-                    if contended_queue.has_contended_task(&task_id) {
-                        address_book.uncontended_task_ids.insert(task_id, ());
+                let maybe_task = l.heaviest_uncontended.load_full();
+                if let Some(task) = maybe_task {
+                    if task.currently_contended() {
+                        address_book.uncontended_task_ids.insert(task.unique_weight, task);
                     }
                 }
             }

From bdf3177e39cc71ec16dae81463551c8bf1d4ba39 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 23 Aug 2022 11:22:14 +0900
Subject: [PATCH 0992/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index b49504c0b12f08..c850b7b8ae0d84 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -646,7 +646,7 @@ impl ScheduleStage {
     */
 
     #[inline(never)]
-    fn get_heaviest_from_contended<'a>(address_book: &'a mut AddressBook) -> Option<std::collections::btree_map::OccupiedEntry<'a, UniqueWeight, ()>> {
+    fn get_heaviest_from_contended<'a>(address_book: &'a mut AddressBook) -> Option<std::collections::btree_map::OccupiedEntry<'a, UniqueWeight, TaskInQueue>> {
         address_book.uncontended_task_ids.last_entry()
     }
 

From a4e1864bcdff6686b4bf32328f786be7aedb5c23 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 23 Aug 2022 11:23:00 +0900
Subject: [PATCH 0993/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index c850b7b8ae0d84..dad555e43d3c55 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -886,7 +886,7 @@ impl ScheduleStage {
                             if tracker.is_fulfilled() {
                                 trace!("provisioning tracker tick: {} => {} (!)", tracker.prev_count(), tracker.count());
                                 tracker_entry.remove();
-                                address_book.fulfilled_provisional_task_ids.insert(task_id, ());
+                                panic!()//address_book.fulfilled_provisional_task_ids.insert(task_id, ());
                             } else {
                                 trace!("provisioning tracker tick: {} => {}", tracker.prev_count(), tracker.count());
                             }

From 270b73b895f819afd288a490df290fc6dc395b52 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 23 Aug 2022 11:24:05 +0900
Subject: [PATCH 0994/3199] save

---
 scheduler/src/lib.rs                    | 5 +++--
 transaction-scheduler-bench/src/main.rs | 2 +-
 2 files changed, 4 insertions(+), 3 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index dad555e43d3c55..cb89da107c226e 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -457,14 +457,15 @@ struct Bundle {
 
 #[derive(Debug)]
 pub struct Task {
+    unique_weight: UniqueWeight,
     pub tx: Box<(SanitizedTransaction, Vec<LockAttempt>)>, // actually should be Bundle
     pub contention_count: usize,
     pub uncontended: std::sync::atomic::AtomicUsize,
 }
 
 impl Task {
-    pub fn new_for_queue(tx: Box<(SanitizedTransaction, Vec<LockAttempt>)>) -> std::sync::Arc<Self> {
-        TaskInQueue::new(Self { tx, contention_count: 0, uncontended: Default::default() })
+    pub fn new_for_queue(unique_weight: UniqueWeight, tx: Box<(SanitizedTransaction, Vec<LockAttempt>)>) -> std::sync::Arc<Self> {
+        TaskInQueue::new(Self { unique_weight, tx, contention_count: 0, uncontended: Default::default() })
     }
 
     pub fn currently_contended(&self) -> bool {
diff --git a/transaction-scheduler-bench/src/main.rs b/transaction-scheduler-bench/src/main.rs
index 34a55dda95a6f1..d77cafc9d28270 100644
--- a/transaction-scheduler-bench/src/main.rs
+++ b/transaction-scheduler-bench/src/main.rs
@@ -471,7 +471,7 @@ fn send_packets(
             for v in vv {
                 let p = solana_scheduler::get_transaction_priority_details(&v.0);
                 let p = (p << 32) | (rng.gen::<u64>() & 0x0000_0000_ffff_ffff);
-                let t = solana_scheduler::Task::new_for_queue(v);
+                let t = solana_scheduler::Task::new_for_queue(p, v);
                 for lock_attempt in v.1.iter() {
                     lock_attempt.target.page_ref().contended_unique_weights.insert_task_id(p, t);
                 }

From 9b588d0562ab7a0ab8b5c3fc2522ef6bef88a472 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 23 Aug 2022 11:25:02 +0900
Subject: [PATCH 0995/3199] save

---
 transaction-scheduler-bench/src/main.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/transaction-scheduler-bench/src/main.rs b/transaction-scheduler-bench/src/main.rs
index d77cafc9d28270..ff9e75be4fc4d2 100644
--- a/transaction-scheduler-bench/src/main.rs
+++ b/transaction-scheduler-bench/src/main.rs
@@ -359,7 +359,7 @@ fn handle_transaction_batch(
                 }
             }
             if found {
-                lock_attempt.heaviest_uncontended.store(*(task_cursor.key()), std::sync::atomic::Ordering::SeqCst);
+                lock_attempt.heaviest_uncontended.store(Some(task_cursor.value()));
             }
         }
     }

From 3eb258e52eab72af5e228105ae869d1423108799 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 23 Aug 2022 11:25:32 +0900
Subject: [PATCH 0996/3199] save

---
 transaction-scheduler-bench/src/main.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/transaction-scheduler-bench/src/main.rs b/transaction-scheduler-bench/src/main.rs
index ff9e75be4fc4d2..9e583caad9bd87 100644
--- a/transaction-scheduler-bench/src/main.rs
+++ b/transaction-scheduler-bench/src/main.rs
@@ -359,7 +359,7 @@ fn handle_transaction_batch(
                 }
             }
             if found {
-                lock_attempt.heaviest_uncontended.store(Some(task_cursor.value()));
+                lock_attempt.heaviest_uncontended.store(Some(TaskInQueue::clone(task_cursor.value())));
             }
         }
     }

From 3934811d5e7d77111e59d410133ac22f40bf8f5c Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 23 Aug 2022 11:25:43 +0900
Subject: [PATCH 0997/3199] save

---
 transaction-scheduler-bench/src/main.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/transaction-scheduler-bench/src/main.rs b/transaction-scheduler-bench/src/main.rs
index 9e583caad9bd87..9e8bce7d5a0c0d 100644
--- a/transaction-scheduler-bench/src/main.rs
+++ b/transaction-scheduler-bench/src/main.rs
@@ -359,7 +359,7 @@ fn handle_transaction_batch(
                 }
             }
             if found {
-                lock_attempt.heaviest_uncontended.store(Some(TaskInQueue::clone(task_cursor.value())));
+                lock_attempt.heaviest_uncontended.store(Some(solana_scheduler::TaskInQueue::clone(task_cursor.value())));
             }
         }
     }

From 61f8414c6f69689a1ea5717cf9f3829cc61072da Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 23 Aug 2022 11:25:56 +0900
Subject: [PATCH 0998/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index cb89da107c226e..07b8d53c2913ac 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -490,7 +490,7 @@ pub struct TaskQueue {
     tasks: std::sync::Arc<dashmap::DashMap<UniqueWeight, TaskInQueue>>,
 }
 
-type TaskInQueue = std::sync::Arc<Task>;
+pub type TaskInQueue = std::sync::Arc<Task>;
 //type TaskQueueEntry<'a> = im::ordmap::Entry<'a, UniqueWeight, TaskInQueue>;
 //type TaskQueueOccupiedEntry<'a> = im::ordmap::OccupiedEntry<'a, UniqueWeight, TaskInQueue>;
 //type TaskQueueEntry<'a> = im::hashmap::Entry<'a, UniqueWeight, TaskInQueue, std::collections::hash_map::RandomState>;

From c2ba0d404aaa9456fdd86cee66b20886481a780d Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 23 Aug 2022 11:26:30 +0900
Subject: [PATCH 0999/3199] save

---
 transaction-scheduler-bench/src/main.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/transaction-scheduler-bench/src/main.rs b/transaction-scheduler-bench/src/main.rs
index 9e8bce7d5a0c0d..2d7ce6b36e373d 100644
--- a/transaction-scheduler-bench/src/main.rs
+++ b/transaction-scheduler-bench/src/main.rs
@@ -473,7 +473,7 @@ fn send_packets(
                 let p = (p << 32) | (rng.gen::<u64>() & 0x0000_0000_ffff_ffff);
                 let t = solana_scheduler::Task::new_for_queue(p, v);
                 for lock_attempt in v.1.iter() {
-                    lock_attempt.target.page_ref().contended_unique_weights.insert_task_id(p, t);
+                    lock_attempt.target.page_ref().contended_unique_weights.insert_task_id(p, solana_scheduler::TaskInQueue::clone(t));
                 }
                 packet_batch_sender.send(solana_scheduler::Multiplexed::FromPrevious((p, t))).unwrap();
             }

From 7c40a3125e610cfa2dd667d45161ef64ac64e611 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 23 Aug 2022 11:26:38 +0900
Subject: [PATCH 1000/3199] save

---
 transaction-scheduler-bench/src/main.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/transaction-scheduler-bench/src/main.rs b/transaction-scheduler-bench/src/main.rs
index 2d7ce6b36e373d..6caf2459b6808a 100644
--- a/transaction-scheduler-bench/src/main.rs
+++ b/transaction-scheduler-bench/src/main.rs
@@ -473,7 +473,7 @@ fn send_packets(
                 let p = (p << 32) | (rng.gen::<u64>() & 0x0000_0000_ffff_ffff);
                 let t = solana_scheduler::Task::new_for_queue(p, v);
                 for lock_attempt in v.1.iter() {
-                    lock_attempt.target.page_ref().contended_unique_weights.insert_task_id(p, solana_scheduler::TaskInQueue::clone(t));
+                    lock_attempt.target.page_ref().contended_unique_weights.insert_task_id(p, solana_scheduler::TaskInQueue::clone(&t));
                 }
                 packet_batch_sender.send(solana_scheduler::Multiplexed::FromPrevious((p, t))).unwrap();
             }

From ec0a89b4bd02b6c8c08dd20251e47833f57ad387 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 23 Aug 2022 11:27:00 +0900
Subject: [PATCH 1001/3199] save

---
 transaction-scheduler-bench/src/main.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/transaction-scheduler-bench/src/main.rs b/transaction-scheduler-bench/src/main.rs
index 6caf2459b6808a..7bcc7a4eccf877 100644
--- a/transaction-scheduler-bench/src/main.rs
+++ b/transaction-scheduler-bench/src/main.rs
@@ -472,7 +472,7 @@ fn send_packets(
                 let p = solana_scheduler::get_transaction_priority_details(&v.0);
                 let p = (p << 32) | (rng.gen::<u64>() & 0x0000_0000_ffff_ffff);
                 let t = solana_scheduler::Task::new_for_queue(p, v);
-                for lock_attempt in v.1.iter() {
+                for lock_attempt in t.tx.1.iter() {
                     lock_attempt.target.page_ref().contended_unique_weights.insert_task_id(p, solana_scheduler::TaskInQueue::clone(&t));
                 }
                 packet_batch_sender.send(solana_scheduler::Multiplexed::FromPrevious((p, t))).unwrap();

From d7073b2c59c0285e07dda84be2b35a92a2cfb1fa Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 23 Aug 2022 11:29:43 +0900
Subject: [PATCH 1002/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 07b8d53c2913ac..28667356fdbfd1 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -212,7 +212,7 @@ type AddressMapEntry<'a> = dashmap::mapref::entry::Entry<'a, Pubkey, PageRc>;
 pub struct AddressBook {
     book: AddressMap,
     uncontended_task_ids: WeightedTaskIds,
-    provisioning_trackers: std::collections::HashMap<UniqueWeight, ProvisioningTracker>, 
+    provisioning_trackers: std::collections::HashMap<UniqueWeight, (ProvisioningTracker, TaskInQueue)>, 
     fulfilled_provisional_task_ids: WeightedTaskIds,
 }
 

From 43fd470185bbbed77a88e13788a06c6632286634 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 23 Aug 2022 11:30:37 +0900
Subject: [PATCH 1003/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 28667356fdbfd1..ee525245a6cf37 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -780,7 +780,7 @@ impl ScheduleStage {
                 Self::finalize_lock_for_provisional_execution(
                     address_book,
                     &unique_weight,
-                    &mut next_task.tx.1,
+                    &mut next_task,
                     provisional_count,
                 );
 

From 2d3bfa97675646ef1ec5a8d7cd50a4a8d1cce267 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 23 Aug 2022 11:31:51 +0900
Subject: [PATCH 1004/3199] save

---
 scheduler/src/lib.rs | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index ee525245a6cf37..e5e1e326322b3b 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -728,8 +728,8 @@ impl ScheduleStage {
             trace!("pop loop iteration");
             let from_runnable = reborrowed_contended_queue.is_some();
             let unique_weight = *queue_entry.key();
-            let mut next_task = queue_entry.get_mut();
-            let next_task = unsafe { TaskInQueue::get_mut_unchecked(&mut next_task) };
+            let mut arc_next_task = queue_entry.get_mut();
+            let next_task = unsafe { TaskInQueue::get_mut_unchecked(&mut arc_next_task) };
             let message_hash = next_task.tx.0.message_hash();
 
             // plumb message_hash into StatusCache or implmenent our own for duplicate tx
@@ -777,10 +777,11 @@ impl ScheduleStage {
                 assert!(!from_runnable);
                 let lock_count = next_task.tx.1.len();
                 trace!("provisional exec: [{}/{}]", provisional_count, lock_count);
+                drop(next_task);
                 Self::finalize_lock_for_provisional_execution(
                     address_book,
                     &unique_weight,
-                    &mut next_task,
+                    &mut next_task_arc,
                     provisional_count,
                 );
 

From d016d292df636bb100d251de7bcfacdf182ef2b1 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 23 Aug 2022 11:32:11 +0900
Subject: [PATCH 1005/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index e5e1e326322b3b..aff550817f60a4 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -781,7 +781,7 @@ impl ScheduleStage {
                 Self::finalize_lock_for_provisional_execution(
                     address_book,
                     &unique_weight,
-                    &mut next_task_arc,
+                    &mut arc_next_task,
                     provisional_count,
                 );
 

From 082da66686a756bb841a8322af39d3a39355c432 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 23 Aug 2022 11:33:01 +0900
Subject: [PATCH 1006/3199] save

---
 scheduler/src/lib.rs | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index aff550817f60a4..e2d27442ecdaa4 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -777,13 +777,14 @@ impl ScheduleStage {
                 assert!(!from_runnable);
                 let lock_count = next_task.tx.1.len();
                 trace!("provisional exec: [{}/{}]", provisional_count, lock_count);
-                drop(next_task);
                 Self::finalize_lock_for_provisional_execution(
                     address_book,
                     &unique_weight,
-                    &mut arc_next_task,
+                    &mut next_task,
                     provisional_count,
                 );
+                drop(next_task);
+                address_book.provisioning_trackers.insert(*unique_weight, (ProvisioningTracker::new(provisional_count), arc_next_task));
 
                 return None;
                 continue;
@@ -839,7 +840,6 @@ impl ScheduleStage {
                 }
             }
         }
-        address_book.provisioning_trackers.insert(*unique_weight, ProvisioningTracker::new(provisional_count));
         trace!("provisioning_trackers: {}", address_book.provisioning_trackers.len());
     }
 

From faffb9321c0db0a397b92d06e5e9e20c5182fc84 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 23 Aug 2022 11:33:21 +0900
Subject: [PATCH 1007/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index e2d27442ecdaa4..634fab4af09484 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -780,7 +780,7 @@ impl ScheduleStage {
                 Self::finalize_lock_for_provisional_execution(
                     address_book,
                     &unique_weight,
-                    &mut next_task,
+                    &mut next_task.tx.1,
                     provisional_count,
                 );
                 drop(next_task);

From 1c60da3a967497f4a4ba1239ec3d26141ec849f9 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 23 Aug 2022 11:33:34 +0900
Subject: [PATCH 1008/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 634fab4af09484..8411473fdfbc14 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -784,7 +784,7 @@ impl ScheduleStage {
                     provisional_count,
                 );
                 drop(next_task);
-                address_book.provisioning_trackers.insert(*unique_weight, (ProvisioningTracker::new(provisional_count), arc_next_task));
+                address_book.provisioning_trackers.insert(unique_weight, (ProvisioningTracker::new(provisional_count), arc_next_task));
 
                 return None;
                 continue;

From 38b524428110beaf6f9d53006de45537f57c7cd0 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 23 Aug 2022 11:34:01 +0900
Subject: [PATCH 1009/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 8411473fdfbc14..1a63ac710c28a0 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -784,7 +784,7 @@ impl ScheduleStage {
                     provisional_count,
                 );
                 drop(next_task);
-                address_book.provisioning_trackers.insert(unique_weight, (ProvisioningTracker::new(provisional_count), arc_next_task));
+                address_book.provisioning_trackers.insert(unique_weight, (ProvisioningTracker::new(provisional_count), TaskInQueue::clone(arc_next_task)));
 
                 return None;
                 continue;

From 18b16971be276c74e437f41665287a553d50f21c Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 23 Aug 2022 11:34:27 +0900
Subject: [PATCH 1010/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 1a63ac710c28a0..a88910138bd576 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -883,7 +883,7 @@ impl ScheduleStage {
                 for task_id in std::mem::take(&mut page.provisional_task_ids).into_iter() {
                     match address_book.provisioning_trackers.entry(task_id) {
                         std::collections::hash_map::Entry::Occupied(mut tracker_entry) => {
-                            let tracker = tracker_entry.get_mut();
+                            let (tracker, task) = tracker_entry.get_mut();
                             tracker.tick();
                             if tracker.is_fulfilled() {
                                 trace!("provisioning tracker tick: {} => {} (!)", tracker.prev_count(), tracker.count());

From 16f6bfbc9fc6c015f546585397f9526c32e2764e Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 23 Aug 2022 11:34:48 +0900
Subject: [PATCH 1011/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index a88910138bd576..34b59d6f36d3bc 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -888,7 +888,7 @@ impl ScheduleStage {
                             if tracker.is_fulfilled() {
                                 trace!("provisioning tracker tick: {} => {} (!)", tracker.prev_count(), tracker.count());
                                 tracker_entry.remove();
-                                panic!()//address_book.fulfilled_provisional_task_ids.insert(task_id, ());
+                                address_book.fulfilled_provisional_task_ids.insert(task_id, task);
                             } else {
                                 trace!("provisioning tracker tick: {} => {}", tracker.prev_count(), tracker.count());
                             }

From 7df8ecdf36627bdafd107d11cf3d80e5bc666fd9 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 23 Aug 2022 11:35:11 +0900
Subject: [PATCH 1012/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 34b59d6f36d3bc..5788bbbffd30d7 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -887,7 +887,7 @@ impl ScheduleStage {
                             tracker.tick();
                             if tracker.is_fulfilled() {
                                 trace!("provisioning tracker tick: {} => {} (!)", tracker.prev_count(), tracker.count());
-                                tracker_entry.remove();
+                                let (tracker, task) = tracker_entry.remove();
                                 address_book.fulfilled_provisional_task_ids.insert(task_id, task);
                             } else {
                                 trace!("provisioning tracker tick: {} => {}", tracker.prev_count(), tracker.count());

From 8ff1b514cd0b5963aace186467a1f6447c0f6203 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 23 Aug 2022 11:40:08 +0900
Subject: [PATCH 1013/3199] save

---
 scheduler/src/lib.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 5788bbbffd30d7..7459723824904a 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -484,10 +484,10 @@ impl Task {
 // RunnableQueue, ContendedQueue?
 #[derive(Default, Debug, Clone)]
 pub struct TaskQueue {
-    //tasks: std::collections::BTreeMap<UniqueWeight, Task>,
+    tasks: std::collections::BTreeMap<UniqueWeight, TaskInQueue>,
     //tasks: im::OrdMap<UniqueWeight, TaskInQueue>,
     //tasks: im::HashMap<UniqueWeight, TaskInQueue>,
-    tasks: std::sync::Arc<dashmap::DashMap<UniqueWeight, TaskInQueue>>,
+    //tasks: std::sync::Arc<dashmap::DashMap<UniqueWeight, TaskInQueue>>,
 }
 
 pub type TaskInQueue = std::sync::Arc<Task>;

From 19c217a5ccf1f7618e4c5fbb24f17b5b4b7674d7 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 23 Aug 2022 11:40:39 +0900
Subject: [PATCH 1014/3199] save

---
 scheduler/src/lib.rs | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 7459723824904a..464167111611c6 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -495,8 +495,10 @@ pub type TaskInQueue = std::sync::Arc<Task>;
 //type TaskQueueOccupiedEntry<'a> = im::ordmap::OccupiedEntry<'a, UniqueWeight, TaskInQueue>;
 //type TaskQueueEntry<'a> = im::hashmap::Entry<'a, UniqueWeight, TaskInQueue, std::collections::hash_map::RandomState>;
 //type TaskQueueOccupiedEntry<'a> = im::hashmap::OccupiedEntry<'a, UniqueWeight, TaskInQueue, std::collections::hash_map::RandomState>;
-type TaskQueueEntry<'a> = dashmap::mapref::entry::Entry<'a, UniqueWeight, TaskInQueue>;
-type TaskQueueOccupiedEntry<'a> = dashmap::mapref::entry::OccupiedEntry<'a, UniqueWeight, TaskInQueue, std::collections::hash_map::RandomState>;
+//type TaskQueueEntry<'a> = dashmap::mapref::entry::Entry<'a, UniqueWeight, TaskInQueue>;
+//type TaskQueueOccupiedEntry<'a> = dashmap::mapref::entry::OccupiedEntry<'a, UniqueWeight, TaskInQueue, std::collections::hash_map::RandomState>;
+type TaskQueueEntry<'a> = td::collections::btree_map::Entry'a, UniqueWeight, TaskInQueue>;
+type TaskQueueOccupiedEntry<'a> = std::collections::btree_map::EntryEntry<'a, UniqueWeight, TaskInQueue>;
 
 impl TaskQueue {
     #[inline(never)]

From 4787c8c997eabed4c674dd8067b2a4ca7189a816 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 23 Aug 2022 11:40:47 +0900
Subject: [PATCH 1015/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 464167111611c6..19ee0d76589157 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -497,7 +497,7 @@ pub type TaskInQueue = std::sync::Arc<Task>;
 //type TaskQueueOccupiedEntry<'a> = im::hashmap::OccupiedEntry<'a, UniqueWeight, TaskInQueue, std::collections::hash_map::RandomState>;
 //type TaskQueueEntry<'a> = dashmap::mapref::entry::Entry<'a, UniqueWeight, TaskInQueue>;
 //type TaskQueueOccupiedEntry<'a> = dashmap::mapref::entry::OccupiedEntry<'a, UniqueWeight, TaskInQueue, std::collections::hash_map::RandomState>;
-type TaskQueueEntry<'a> = td::collections::btree_map::Entry'a, UniqueWeight, TaskInQueue>;
+type TaskQueueEntry<'a> = td::collections::btree_map::Entry<'a, UniqueWeight, TaskInQueue>;
 type TaskQueueOccupiedEntry<'a> = std::collections::btree_map::EntryEntry<'a, UniqueWeight, TaskInQueue>;
 
 impl TaskQueue {

From 32de53f3683f00396f09247fe0abd0bd8891e177 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 23 Aug 2022 11:40:54 +0900
Subject: [PATCH 1016/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 19ee0d76589157..e80ed1d9564a92 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -497,7 +497,7 @@ pub type TaskInQueue = std::sync::Arc<Task>;
 //type TaskQueueOccupiedEntry<'a> = im::hashmap::OccupiedEntry<'a, UniqueWeight, TaskInQueue, std::collections::hash_map::RandomState>;
 //type TaskQueueEntry<'a> = dashmap::mapref::entry::Entry<'a, UniqueWeight, TaskInQueue>;
 //type TaskQueueOccupiedEntry<'a> = dashmap::mapref::entry::OccupiedEntry<'a, UniqueWeight, TaskInQueue, std::collections::hash_map::RandomState>;
-type TaskQueueEntry<'a> = td::collections::btree_map::Entry<'a, UniqueWeight, TaskInQueue>;
+type TaskQueueEntry<'a> = std::collections::btree_map::Entry<'a, UniqueWeight, TaskInQueue>;
 type TaskQueueOccupiedEntry<'a> = std::collections::btree_map::EntryEntry<'a, UniqueWeight, TaskInQueue>;
 
 impl TaskQueue {

From 4f78161a4de1e85227ed3d72ce6c80ba65a97cb0 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 23 Aug 2022 11:41:24 +0900
Subject: [PATCH 1017/3199] save

---
 scheduler/src/lib.rs | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index e80ed1d9564a92..1c44d723910a65 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -533,13 +533,14 @@ impl TaskQueue {
     fn heaviest_entry_to_execute(
         &mut self,
     ) -> Option<TaskQueueOccupiedEntry<'_>> {
-        //panic!()//self.tasks.last_entry()
+        //panic!()
+        self.tasks.last_entry()
         //let k = self.tasks.get_max().map(|(k, _v)| *k);
         //panic!();
         //let k = self.tasks.iter().next().map(|(k, _v)| *k);
-        let k = self.tasks.iter().next().map(|r| *r.key());
+        //let k = self.tasks.iter().next().map(|r| *r.key());
 
-        k.map(|k| self.entry_to_execute(k).unwrap())
+        //k.map(|k| self.entry_to_execute(k).unwrap())
     }
 
     fn task_count(&self) -> usize {

From 68b0b1e9d10f9bd76bd10003b71ae28c42114d8e Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 23 Aug 2022 11:41:47 +0900
Subject: [PATCH 1018/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 1c44d723910a65..3c737673e15a1d 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -498,7 +498,7 @@ pub type TaskInQueue = std::sync::Arc<Task>;
 //type TaskQueueEntry<'a> = dashmap::mapref::entry::Entry<'a, UniqueWeight, TaskInQueue>;
 //type TaskQueueOccupiedEntry<'a> = dashmap::mapref::entry::OccupiedEntry<'a, UniqueWeight, TaskInQueue, std::collections::hash_map::RandomState>;
 type TaskQueueEntry<'a> = std::collections::btree_map::Entry<'a, UniqueWeight, TaskInQueue>;
-type TaskQueueOccupiedEntry<'a> = std::collections::btree_map::EntryEntry<'a, UniqueWeight, TaskInQueue>;
+type TaskQueueOccupiedEntry<'a> = std::collections::btree_map::OccupiedEntry<'a, UniqueWeight, TaskInQueue>;
 
 impl TaskQueue {
     #[inline(never)]

From 5ec2531d57ec2870c7ff0c69710d82d0562010c6 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 23 Aug 2022 11:44:09 +0900
Subject: [PATCH 1019/3199] save

---
 scheduler/src/lib.rs | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 3c737673e15a1d..1b32f5b797802b 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -660,8 +660,8 @@ impl ScheduleStage {
         contended_queue: &'a mut TaskQueue,
         address_book: &mut AddressBook,
     ) -> Option<(
-        Option<&'a mut TaskQueue>,
-        TaskQueueOccupiedEntry<'a>,
+        bool,
+        &'a TaskInQueue,
     )> {
         match (
             runnable_queue.heaviest_entry_to_execute(),
@@ -669,7 +669,7 @@ impl ScheduleStage {
         ) {
             (Some(heaviest_runnable_entry), None) => {
                 trace!("select: runnable only");
-                return Some((Some(contended_queue), heaviest_runnable_entry))
+                return Some((true, heaviest_runnable_entry.get()))
             }
             (None, Some(weight_from_contended)) => {
                 trace!("select: contended only");

From 7f9dfab54d71705e96baa1793c2d2c901c939416 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 23 Aug 2022 11:44:47 +0900
Subject: [PATCH 1020/3199] save

---
 scheduler/src/lib.rs | 8 ++------
 1 file changed, 2 insertions(+), 6 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 1b32f5b797802b..aacb4ce700e6e4 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -674,13 +674,9 @@ impl ScheduleStage {
             (None, Some(weight_from_contended)) => {
                 trace!("select: contended only");
                 let uw = *weight_from_contended.key();
-                weight_from_contended.remove();
+                let t = weight_from_contended.remove();
 
-                if let Some (queue_entry) = contended_queue.entry_to_execute(uw) {
-                    return Some(( None, queue_entry))
-                } else {
-                    unreachable!();
-                }
+                return Some(( false, t));
             },
             (Some(heaviest_runnable_entry), Some(weight_from_contended)) => {
                 let weight_from_runnable = heaviest_runnable_entry.key();

From a81af483823cf818dd86ad013540eebfcad37add Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 23 Aug 2022 11:45:05 +0900
Subject: [PATCH 1021/3199] save

---
 scheduler/src/lib.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index aacb4ce700e6e4..2fd9b7780ba4ca 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -661,7 +661,7 @@ impl ScheduleStage {
         address_book: &mut AddressBook,
     ) -> Option<(
         bool,
-        &'a TaskInQueue,
+        TaskInQueue,
     )> {
         match (
             runnable_queue.heaviest_entry_to_execute(),
@@ -669,7 +669,7 @@ impl ScheduleStage {
         ) {
             (Some(heaviest_runnable_entry), None) => {
                 trace!("select: runnable only");
-                return Some((true, heaviest_runnable_entry.get()))
+                return Some((true, TaskInQueue::clone(heaviest_runnable_entry.get())))
             }
             (None, Some(weight_from_contended)) => {
                 trace!("select: contended only");

From b10cdd9787a3499e70310d11854c04d1928d0b0d Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 23 Aug 2022 11:46:05 +0900
Subject: [PATCH 1022/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 2fd9b7780ba4ca..3cf46112c464b4 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -669,7 +669,7 @@ impl ScheduleStage {
         ) {
             (Some(heaviest_runnable_entry), None) => {
                 trace!("select: runnable only");
-                return Some((true, TaskInQueue::clone(heaviest_runnable_entry.get())))
+                return Some((Some(heaviest_runnable_entry.get()), None))
             }
             (None, Some(weight_from_contended)) => {
                 trace!("select: contended only");

From cc90e3cb2303748dda8bd309c863ea98aa9d3cdc Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 23 Aug 2022 11:46:51 +0900
Subject: [PATCH 1023/3199] save

---
 scheduler/src/lib.rs | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 3cf46112c464b4..9c50e5c8c7b845 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -669,14 +669,15 @@ impl ScheduleStage {
         ) {
             (Some(heaviest_runnable_entry), None) => {
                 trace!("select: runnable only");
-                return Some((Some(heaviest_runnable_entry.get()), None))
+                let t = heaviest_runnable_entry.remove();
+                return Some((true, t))
             }
             (None, Some(weight_from_contended)) => {
                 trace!("select: contended only");
                 let uw = *weight_from_contended.key();
                 let t = weight_from_contended.remove();
 
-                return Some(( false, t));
+                return Some(( false, t))
             },
             (Some(heaviest_runnable_entry), Some(weight_from_contended)) => {
                 let weight_from_runnable = heaviest_runnable_entry.key();

From b0d1379549915a891283aaaecc4bd33dd7c55dde Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 23 Aug 2022 11:47:35 +0900
Subject: [PATCH 1024/3199] save

---
 scheduler/src/lib.rs | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 9c50e5c8c7b845..df60a5de0853b3 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -685,8 +685,11 @@ impl ScheduleStage {
 
                 if true || weight_from_runnable > uw {
                     trace!("select: runnable > contended");
-                    return Some((Some(contended_queue), heaviest_runnable_entry))
+                    let t = heaviest_runnable_entry.remove();
+                    return Some((true, t))
                 } else if false && uw > weight_from_runnable {
+                    panic!();
+                    /*
                     trace!("select: contended > runnnable");
                     let uw = *uw;
                     weight_from_contended.remove();
@@ -695,6 +698,7 @@ impl ScheduleStage {
                     } else {
                         unreachable!();
                     }
+                    */
                 } else {
                     unreachable!(
                         "identical unique weights shouldn't exist in both runnable and contended"

From 23d134f926f35351dc44637aa40d4dd7729fb58e Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 23 Aug 2022 11:48:03 +0900
Subject: [PATCH 1025/3199] save

---
 scheduler/src/lib.rs | 6 +-----
 1 file changed, 1 insertion(+), 5 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index df60a5de0853b3..b31fe29f9ff705 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -728,9 +728,8 @@ impl ScheduleStage {
 
         trace!("pop begin");
         loop {
-        if let Some((reborrowed_contended_queue, mut queue_entry)) = Self::select_next_task(runnable_queue, contended_queue, address_book) {
+        if let Some((from_runnable, mut queue_entry)) = Self::select_next_task(runnable_queue, contended_queue, address_book) {
             trace!("pop loop iteration");
-            let from_runnable = reborrowed_contended_queue.is_some();
             let unique_weight = *queue_entry.key();
             let mut arc_next_task = queue_entry.get_mut();
             let next_task = unsafe { TaskInQueue::get_mut_unchecked(&mut arc_next_task) };
@@ -763,9 +762,6 @@ impl ScheduleStage {
                     trace!("move to contended due to lock failure [{}/{}/{}]", unlockable_count, provisional_count, lock_count);
                     let task = queue_entry.remove();
                     task.mark_as_contended();
-                    reborrowed_contended_queue
-                        .unwrap()
-                        .add_to_schedule(unique_weight, task);
                     // maybe run lightweight prune logic on contended_queue here.
                 } else {
                     trace!("relock failed [{}/{}/{}]; remains in contended: {:?} contention: {}", unlockable_count, provisional_count, lock_count, &unique_weight, next_task.contention_count);

From e18f4820bfad5814020a634a00953db72c72efc9 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 23 Aug 2022 11:49:17 +0900
Subject: [PATCH 1026/3199] save

---
 scheduler/src/lib.rs | 7 ++-----
 1 file changed, 2 insertions(+), 5 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index b31fe29f9ff705..12b3580679f1a1 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -728,11 +728,10 @@ impl ScheduleStage {
 
         trace!("pop begin");
         loop {
-        if let Some((from_runnable, mut queue_entry)) = Self::select_next_task(runnable_queue, contended_queue, address_book) {
+        if let Some((from_runnable, mut arc_next_task)) = Self::select_next_task(runnable_queue, contended_queue, address_book) {
             trace!("pop loop iteration");
-            let unique_weight = *queue_entry.key();
-            let mut arc_next_task = queue_entry.get_mut();
             let next_task = unsafe { TaskInQueue::get_mut_unchecked(&mut arc_next_task) };
+            let unique_weight = *next_task.unique_weight;
             let message_hash = next_task.tx.0.message_hash();
 
             // plumb message_hash into StatusCache or implmenent our own for duplicate tx
@@ -760,7 +759,6 @@ impl ScheduleStage {
 
                 if from_runnable {
                     trace!("move to contended due to lock failure [{}/{}/{}]", unlockable_count, provisional_count, lock_count);
-                    let task = queue_entry.remove();
                     task.mark_as_contended();
                     // maybe run lightweight prune logic on contended_queue here.
                 } else {
@@ -796,7 +794,6 @@ impl ScheduleStage {
                 &unique_weight,
                 &mut next_task.tx.1,
             );
-            let task = queue_entry.remove();
             return Some((unique_weight, task));
         } else {
             break;

From 93bb241abf02f946463fe5f447a5857928c2bc2e Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 23 Aug 2022 11:49:49 +0900
Subject: [PATCH 1027/3199] save

---
 scheduler/src/lib.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 12b3580679f1a1..b8295831f70cf0 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -731,7 +731,7 @@ impl ScheduleStage {
         if let Some((from_runnable, mut arc_next_task)) = Self::select_next_task(runnable_queue, contended_queue, address_book) {
             trace!("pop loop iteration");
             let next_task = unsafe { TaskInQueue::get_mut_unchecked(&mut arc_next_task) };
-            let unique_weight = *next_task.unique_weight;
+            let unique_weight = next_task.unique_weight;
             let message_hash = next_task.tx.0.message_hash();
 
             // plumb message_hash into StatusCache or implmenent our own for duplicate tx
@@ -782,7 +782,7 @@ impl ScheduleStage {
                     provisional_count,
                 );
                 drop(next_task);
-                address_book.provisioning_trackers.insert(unique_weight, (ProvisioningTracker::new(provisional_count), TaskInQueue::clone(arc_next_task)));
+                address_book.provisioning_trackers.insert(unique_weight, (ProvisioningTracker::new(provisional_count), arc_next_task));
 
                 return None;
                 continue;

From 63cc5c77db7d3b933f655fb61ff204629d30632d Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 23 Aug 2022 11:50:08 +0900
Subject: [PATCH 1028/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index b8295831f70cf0..de287aad0beae8 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -759,7 +759,7 @@ impl ScheduleStage {
 
                 if from_runnable {
                     trace!("move to contended due to lock failure [{}/{}/{}]", unlockable_count, provisional_count, lock_count);
-                    task.mark_as_contended();
+                    next_task.mark_as_contended();
                     // maybe run lightweight prune logic on contended_queue here.
                 } else {
                     trace!("relock failed [{}/{}/{}]; remains in contended: {:?} contention: {}", unlockable_count, provisional_count, lock_count, &unique_weight, next_task.contention_count);

From 3e1fb0f5fc3ca576e2e4f8d0d2bb69ff8872fa79 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 23 Aug 2022 11:50:25 +0900
Subject: [PATCH 1029/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index de287aad0beae8..4b0abf448c6e39 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -794,7 +794,7 @@ impl ScheduleStage {
                 &unique_weight,
                 &mut next_task.tx.1,
             );
-            return Some((unique_weight, task));
+            return Some((unique_weight, next_task));
         } else {
             break;
         }

From 3f9464a70a822a99f76bf39ca56cfa8ba43e2ea9 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 23 Aug 2022 11:50:34 +0900
Subject: [PATCH 1030/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 4b0abf448c6e39..dce07e24e2e223 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -794,7 +794,7 @@ impl ScheduleStage {
                 &unique_weight,
                 &mut next_task.tx.1,
             );
-            return Some((unique_weight, next_task));
+            return Some((unique_weight, arc_next_task));
         } else {
             break;
         }

From 2d976ee710cea78fa56e1d9a91dbc04793b821e6 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 23 Aug 2022 11:52:14 +0900
Subject: [PATCH 1031/3199] save

---
 scheduler/src/lib.rs | 29 ++++++++++-------------------
 1 file changed, 10 insertions(+), 19 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index dce07e24e2e223..dcdb0b7f8635a6 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -34,7 +34,6 @@ pub struct ExecutionEnvironment {
     pub cu: usize,
     pub unique_weight: UniqueWeight,
     pub task: TaskInQueue,
-    pub checkpointed_contended_queue: TaskQueue,
 }
 
 impl ExecutionEnvironment {
@@ -657,7 +656,6 @@ impl ScheduleStage {
     #[inline(never)]
     fn select_next_task<'a>(
         runnable_queue: &'a mut TaskQueue,
-        contended_queue: &'a mut TaskQueue,
         address_book: &mut AddressBook,
     ) -> Option<(
         bool,
@@ -715,20 +713,17 @@ impl ScheduleStage {
     #[inline(never)]
     fn pop_from_queue_then_lock(
         runnable_queue: &mut TaskQueue,
-        contended_queue: &mut TaskQueue,
         address_book: &mut AddressBook,
         prefer_immediate: bool,
     ) -> Option<(UniqueWeight, TaskInQueue)> {
         if let Some(a) = address_book.fulfilled_provisional_task_ids.pop_last() {
             trace!("expediate pop from provisional queue [rest: {}]", address_book.fulfilled_provisional_task_ids.len());
-            let queue_entry = contended_queue.entry_to_execute(a.0).unwrap();
-            let mut task = queue_entry.remove();
-            return Some((a.0, task));
+            return Some((a.0, a.1));
         }
 
         trace!("pop begin");
         loop {
-        if let Some((from_runnable, mut arc_next_task)) = Self::select_next_task(runnable_queue, contended_queue, address_book) {
+        if let Some((from_runnable, mut arc_next_task)) = Self::select_next_task(runnable_queue, address_book) {
             trace!("pop loop iteration");
             let next_task = unsafe { TaskInQueue::get_mut_unchecked(&mut arc_next_task) };
             let unique_weight = next_task.unique_weight;
@@ -862,7 +857,7 @@ impl ScheduleStage {
     }
 
     #[inline(never)]
-    fn unlock_after_execution(address_book: &mut AddressBook, contended_queue: &TaskQueue, lock_attempts: &mut Vec<LockAttempt>) {
+    fn unlock_after_execution(address_book: &mut AddressBook, lock_attempts: &mut Vec<LockAttempt>) {
         for mut l in lock_attempts.into_iter() {
             let newly_uncontended_while_queued = address_book.reset_lock(&mut l, true);
 
@@ -906,26 +901,23 @@ impl ScheduleStage {
         address_book: &mut AddressBook,
         unique_weight: UniqueWeight,
         task: TaskInQueue,
-        contended_queue: &TaskQueue,
     ) -> Box<ExecutionEnvironment> {
         let mut rng = rand::thread_rng();
         // load account now from AccountsDb
-        let checkpointed_contended_queue = contended_queue.clone();
         task.mark_as_uncontended();
 
         Box::new(ExecutionEnvironment {
             task,
             unique_weight,
             cu: rng.gen_range(3, 1000),
-            checkpointed_contended_queue,
         })
     }
 
     #[inline(never)]
-    fn commit_result(ee: &mut ExecutionEnvironment, address_book: &mut AddressBook, contended_queue: &TaskQueue) {
+    fn commit_result(ee: &mut ExecutionEnvironment, address_book: &mut AddressBook) {
         // do par()-ly?
         let mut task = unsafe { TaskInQueue::get_mut_unchecked(&mut ee.task) };
-        Self::unlock_after_execution(address_book, contended_queue, &mut task.tx.1);
+        Self::unlock_after_execution(address_book, &mut task.tx.1);
         // block-wide qos validation will be done here
         // if error risen..:
         //   don't commit the tx for banking and potentially finish scheduling at block max cu
@@ -938,13 +930,12 @@ impl ScheduleStage {
     #[inline(never)]
     fn schedule_next_execution(
         runnable_queue: &mut TaskQueue,
-        contended_queue: &mut TaskQueue,
         address_book: &mut AddressBook,
         prefer_immediate: bool,
     ) -> Option<Box<ExecutionEnvironment>> {
         let maybe_ee =
-            Self::pop_from_queue_then_lock(runnable_queue, contended_queue, address_book, prefer_immediate)
-                .map(|(uw, t)| Self::prepare_scheduled_execution(address_book, uw, t, contended_queue));
+            Self::pop_from_queue_then_lock(runnable_queue, address_book, prefer_immediate)
+                .map(|(uw, t)| Self::prepare_scheduled_execution(address_book, uw, t));
         maybe_ee
     }
 
@@ -998,7 +989,7 @@ impl ScheduleStage {
                                 trace!("recv from execute: {:?}", processed_execution_environment.unique_weight);
                                 executing_queue_count -= 1;
 
-                                Self::commit_result(&mut processed_execution_environment, address_book, contended_queue);
+                                Self::commit_result(&mut processed_execution_environment, address_book);
                                 // async-ly propagate the result to rpc subsystems
                                 to_next_stage.send(processed_execution_environment).unwrap();
                             }
@@ -1029,7 +1020,7 @@ impl ScheduleStage {
                         trace!("recv from execute: {:?}", processed_execution_environment.unique_weight);
                         executing_queue_count -= 1;
 
-                        Self::commit_result(&mut processed_execution_environment, address_book, contended_queue);
+                        Self::commit_result(&mut processed_execution_environment, address_book);
                         // async-ly propagate the result to rpc subsystems
                         to_next_stage.send(processed_execution_environment).unwrap();
                         if false && from_exec.len() > 0 {
@@ -1063,7 +1054,7 @@ impl ScheduleStage {
 
                 let prefer_immediate = address_book.provisioning_trackers.len()/4 > executing_queue_count;
                 let maybe_ee =
-                    Self::schedule_next_execution(runnable_queue, contended_queue, address_book, prefer_immediate);
+                    Self::schedule_next_execution(runnable_queue, address_book, prefer_immediate);
 
                 if let Some(ee) = maybe_ee {
                     trace!("send to execute");

From 9c4ae885f155712701954e817911bce0b85d85f0 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 23 Aug 2022 12:17:18 +0900
Subject: [PATCH 1032/3199] save

---
 scheduler/src/lib.rs | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index dcdb0b7f8635a6..30c3f252df9257 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -714,6 +714,7 @@ impl ScheduleStage {
     fn pop_from_queue_then_lock(
         runnable_queue: &mut TaskQueue,
         address_book: &mut AddressBook,
+        contended_count: &mut usize,
         prefer_immediate: bool,
     ) -> Option<(UniqueWeight, TaskInQueue)> {
         if let Some(a) = address_book.fulfilled_provisional_task_ids.pop_last() {
@@ -755,6 +756,7 @@ impl ScheduleStage {
                 if from_runnable {
                     trace!("move to contended due to lock failure [{}/{}/{}]", unlockable_count, provisional_count, lock_count);
                     next_task.mark_as_contended();
+                    contended_count += 1;
                     // maybe run lightweight prune logic on contended_queue here.
                 } else {
                     trace!("relock failed [{}/{}/{}]; remains in contended: {:?} contention: {}", unlockable_count, provisional_count, lock_count, &unique_weight, next_task.contention_count);
@@ -931,10 +933,11 @@ impl ScheduleStage {
     fn schedule_next_execution(
         runnable_queue: &mut TaskQueue,
         address_book: &mut AddressBook,
+        contended_count: &mut usize,
         prefer_immediate: bool,
     ) -> Option<Box<ExecutionEnvironment>> {
         let maybe_ee =
-            Self::pop_from_queue_then_lock(runnable_queue, address_book, prefer_immediate)
+            Self::pop_from_queue_then_lock(runnable_queue, contended_count, address_book, prefer_immediate)
                 .map(|(uw, t)| Self::prepare_scheduled_execution(address_book, uw, t));
         maybe_ee
     }
@@ -951,7 +954,6 @@ impl ScheduleStage {
     pub fn run(
         max_executing_queue_count: usize,
         runnable_queue: &mut TaskQueue,
-        contended_queue: &mut TaskQueue,
         address_book: &mut AddressBook,
         from: &crossbeam_channel::Receiver<Multiplexed>,
         from_exec: &crossbeam_channel::Receiver<Box<ExecutionEnvironment>>,
@@ -960,6 +962,7 @@ impl ScheduleStage {
     ) {
         let mut executing_queue_count = 0;
         let mut current_unique_key = u64::max_value();
+        let mut contended_count = 0;
         let (ee_sender, ee_receiver) = crossbeam_channel::unbounded::<Box<ExecutionEnvironment>>();
 
         let (to_next_stage, maybe_jon_handle) = if let Some(to_next_stage) = maybe_to_next_stage {
@@ -975,7 +978,7 @@ impl ScheduleStage {
         };
 
         loop {
-            trace!("schedule_once (from: {}, to: {}, runnnable: {}, contended: {}, (immediate+provisional)/max: ({}+{})/{}) active from contended: {}!", from.len(), to_execute_substage.len(), runnable_queue.task_count(), contended_queue.task_count(), executing_queue_count, address_book.provisioning_trackers.len(), max_executing_queue_count, address_book.uncontended_task_ids.len());
+            trace!("schedule_once (from: {}, to: {}, runnnable: {}, contended: {}, (immediate+provisional)/max: ({}+{})/{}) active from contended: {}!", from.len(), to_execute_substage.len(), runnable_queue.task_count(), contended_count, executing_queue_count, address_book.provisioning_trackers.len(), max_executing_queue_count, address_book.uncontended_task_ids.len());
 
             crossbeam_channel::select! {
                recv(from) -> maybe_from => {

From ed0021f648ad103d1135bf7056a1d5b8b61315b5 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 23 Aug 2022 12:17:47 +0900
Subject: [PATCH 1033/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 30c3f252df9257..ee1a139a2486af 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1057,7 +1057,7 @@ impl ScheduleStage {
 
                 let prefer_immediate = address_book.provisioning_trackers.len()/4 > executing_queue_count;
                 let maybe_ee =
-                    Self::schedule_next_execution(runnable_queue, address_book, prefer_immediate);
+                    Self::schedule_next_execution(runnable_queue, address_book, &mut contended_count, prefer_immediate);
 
                 if let Some(ee) = maybe_ee {
                     trace!("send to execute");

From 4754ae06eda363b9afc10d0ed1ceb440785ac164 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 23 Aug 2022 12:18:07 +0900
Subject: [PATCH 1034/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index ee1a139a2486af..b246ccad1d4e2d 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -937,7 +937,7 @@ impl ScheduleStage {
         prefer_immediate: bool,
     ) -> Option<Box<ExecutionEnvironment>> {
         let maybe_ee =
-            Self::pop_from_queue_then_lock(runnable_queue, contended_count, address_book, prefer_immediate)
+            Self::pop_from_queue_then_lock(runnable_queue, address_book, contended_count, prefer_immediate)
                 .map(|(uw, t)| Self::prepare_scheduled_execution(address_book, uw, t));
         maybe_ee
     }

From 8f81b1efd7b412f5f2f21d070855011852d26982 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 23 Aug 2022 12:18:17 +0900
Subject: [PATCH 1035/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index b246ccad1d4e2d..330f1f4d7a2775 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -756,7 +756,7 @@ impl ScheduleStage {
                 if from_runnable {
                     trace!("move to contended due to lock failure [{}/{}/{}]", unlockable_count, provisional_count, lock_count);
                     next_task.mark_as_contended();
-                    contended_count += 1;
+                    *contended_count += 1;
                     // maybe run lightweight prune logic on contended_queue here.
                 } else {
                     trace!("relock failed [{}/{}/{}]; remains in contended: {:?} contention: {}", unlockable_count, provisional_count, lock_count, &unique_weight, next_task.contention_count);

From 59059e19f0edd102cbf4e679ebe602cd6db884bf Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 23 Aug 2022 12:18:47 +0900
Subject: [PATCH 1036/3199] save

---
 scheduler/src/lib.rs | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 330f1f4d7a2775..62aaa71dc49cb1 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -903,10 +903,12 @@ impl ScheduleStage {
         address_book: &mut AddressBook,
         unique_weight: UniqueWeight,
         task: TaskInQueue,
+        contended_count: &mut usize,
     ) -> Box<ExecutionEnvironment> {
         let mut rng = rand::thread_rng();
         // load account now from AccountsDb
         task.mark_as_uncontended();
+        *contended_count -= 1;
 
         Box::new(ExecutionEnvironment {
             task,
@@ -938,7 +940,7 @@ impl ScheduleStage {
     ) -> Option<Box<ExecutionEnvironment>> {
         let maybe_ee =
             Self::pop_from_queue_then_lock(runnable_queue, address_book, contended_count, prefer_immediate)
-                .map(|(uw, t)| Self::prepare_scheduled_execution(address_book, uw, t));
+                .map(|(uw, t)| Self::prepare_scheduled_execution(address_book, uw, t, contended_count));
         maybe_ee
     }
 

From 406b02bb07901e614fc95c6fce41021f60ee9ff6 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 23 Aug 2022 12:19:07 +0900
Subject: [PATCH 1037/3199] save

---
 transaction-scheduler-bench/src/main.rs | 2 --
 1 file changed, 2 deletions(-)

diff --git a/transaction-scheduler-bench/src/main.rs b/transaction-scheduler-bench/src/main.rs
index 7bcc7a4eccf877..9505c2964236e6 100644
--- a/transaction-scheduler-bench/src/main.rs
+++ b/transaction-scheduler-bench/src/main.rs
@@ -129,12 +129,10 @@ fn spawn_unified_scheduler(
 ) -> JoinHandle<()> {
     std::thread::Builder::new().name("sol-scheduler".to_string()).spawn(move || {
         let mut runnable_queue = solana_scheduler::TaskQueue::default();
-        let mut contended_queue = solana_scheduler::TaskQueue::default();
 
         solana_scheduler::ScheduleStage::run(
             num_execution_threads * 10,
             &mut runnable_queue,
-            &mut contended_queue,
             &mut address_book,
             &packet_batch_receiver.clone(),
             &completed_transaction_receiver,

From 5575d62f28b50efb1b31b25fa05c34680524dd81 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 23 Aug 2022 12:28:38 +0900
Subject: [PATCH 1038/3199] save

---
 ledger-tool/src/main.rs | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/ledger-tool/src/main.rs b/ledger-tool/src/main.rs
index 5ac9eeed399bc4..f2e3b0fbe36335 100644
--- a/ledger-tool/src/main.rs
+++ b/ledger-tool/src/main.rs
@@ -269,7 +269,6 @@ fn output_slot(
     //
     let (post_schedule_env_sender, post_schedule_env_receiver) = crossbeam_channel::unbounded();
     let mut runnable_queue = TaskQueue::default();
-    let mut contended_queue = TaskQueue::default();
     let mut address_book = AddressBook::default();
     let preloader = address_book.preloader();
     let t1 = std::thread::Builder::new()
@@ -278,7 +277,6 @@ fn output_slot(
             ScheduleStage::run(
                 lane_count * lane_channel_factor,
                 &mut runnable_queue,
-                &mut contended_queue,
                 &mut address_book,
                 &muxed_receiver,
                 &post_execute_env_receiver,
@@ -406,15 +404,16 @@ fn output_slot(
                     std::thread::sleep(std::time::Duration::from_micros(10));
                 }
 
+                let t = solana_scheduler::Task::new_for_queue(weight, tx);
                 for lock_attempt in tx.1.iter() {
-                    lock_attempt.target.page_ref().contended_unique_weights.insert_task_id(weight);
+                    lock_attempt.target.page_ref().contended_unique_weights.insert_task_id(weight, solana_scheduler::TaskInQueue::clone(&t));
                 }
 
                 muxed_sender
                     .send(solana_scheduler::Multiplexed::FromPrevious((
                         //Weight { ix: weight },
                         weight,
-                        tx,
+                        t,
                     )))
                     .unwrap();
                 depth.fetch_add(1, Ordering::Relaxed);

From 772ebad153fb002dae4dd05c4681efe69c138c66 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 23 Aug 2022 12:30:26 +0900
Subject: [PATCH 1039/3199] save

---
 ledger-tool/src/main.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/ledger-tool/src/main.rs b/ledger-tool/src/main.rs
index f2e3b0fbe36335..0b014363bab0d2 100644
--- a/ledger-tool/src/main.rs
+++ b/ledger-tool/src/main.rs
@@ -338,7 +338,7 @@ fn output_slot(
                             page.contended_unique_weights.remove_task_id(&uq);
                             if let Some(mut task_cursor) = page.contended_unique_weights.heaviest_task_cursor() {
                                 let mut found = true;
-                                while !ee.checkpointed_contended_queue.has_task(task_cursor.value()) {
+                                while !task_cursor.value().currently_contended() {
                                     if let Some(new_cursor) = task_cursor.prev() {
                                         task_cursor = new_cursor;
                                     } else {
@@ -347,7 +347,7 @@ fn output_slot(
                                     }
                                 }
                                 if found {
-                                    lock_attempt.heaviest_uncontended.store(*(task_cursor.value()), std::sync::atomic::Ordering::SeqCst);
+                                    lock_attempt.heaviest_uncontended.store(Some(solana_scheduler::TaskInQueue::clone(task_cursor.value())));
                                 }
                             }
                         }

From 05970ed30e68268fee061198704dd6d4abfefb41 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 23 Aug 2022 12:31:42 +0900
Subject: [PATCH 1040/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 62aaa71dc49cb1..1020b5dc2fc821 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -454,7 +454,7 @@ struct Bundle {
     // what about bundle1{tx1a, tx2} and bundle2{tx1b, tx2}?
 }
 
-#[derive(Debug)]
+#[derive(Debug, Clone)]
 pub struct Task {
     unique_weight: UniqueWeight,
     pub tx: Box<(SanitizedTransaction, Vec<LockAttempt>)>, // actually should be Bundle

From 4da1bbb35c58216fb79731e2a4f963bfb7e4f3d3 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 23 Aug 2022 12:33:07 +0900
Subject: [PATCH 1041/3199] save

---
 scheduler/src/lib.rs | 11 ++++++++++-
 1 file changed, 10 insertions(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 1020b5dc2fc821..e0672cc1adbf3e 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -454,7 +454,7 @@ struct Bundle {
     // what about bundle1{tx1a, tx2} and bundle2{tx1b, tx2}?
 }
 
-#[derive(Debug, Clone)]
+#[derive(Debug)]
 pub struct Task {
     unique_weight: UniqueWeight,
     pub tx: Box<(SanitizedTransaction, Vec<LockAttempt>)>, // actually should be Bundle
@@ -467,6 +467,15 @@ impl Task {
         TaskInQueue::new(Self { unique_weight, tx, contention_count: 0, uncontended: Default::default() })
     }
 
+    pub fn clone_for_test(&self) {
+        Self {
+            unique_weight: self.unique_weight,
+            tx: self.tx.clone(),
+            contention_count: self.contention_count,
+            uncontended: std::sync::atomic::AtomicUsize::default(),
+        }
+    }
+
     pub fn currently_contended(&self) -> bool {
         self.uncontended.load(std::sync::atomic::Ordering::SeqCst) == 1
     }

From d88babdae9ec461e311ce8435e37532c0f16317c Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 23 Aug 2022 12:33:31 +0900
Subject: [PATCH 1042/3199] save

---
 scheduler/src/lib.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index e0672cc1adbf3e..0d4f0896fb9c35 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -471,8 +471,8 @@ impl Task {
         Self {
             unique_weight: self.unique_weight,
             tx: self.tx.clone(),
-            contention_count: self.contention_count,
-            uncontended: std::sync::atomic::AtomicUsize::default(),
+            contention_count: Default:default()
+            uncontended: Default::default(),
         }
     }
 

From ce0ef817b5b0e833b30de3a849113228cba187b0 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 23 Aug 2022 12:35:39 +0900
Subject: [PATCH 1043/3199] save

---
 scheduler/src/lib.rs | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 0d4f0896fb9c35..7f7e453c8ce322 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -85,6 +85,16 @@ impl LockAttempt {
             //remembered: false,
         }
     }
+
+    pub fn clone_for_test(&self) -> Self {
+        Self {
+            target: self.target.clone,,
+            status: LockStatus::Succeded,
+            requested_usage: self.requested_usage,
+            heaviest_uncontended: Default::default(),
+            //remembered: false,
+        }
+    }
 }
 
 type UsageCount = usize;

From 2d95769f181cb770dcd872e84d162f84e9aedb11 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 23 Aug 2022 12:35:51 +0900
Subject: [PATCH 1044/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 7f7e453c8ce322..ec231b7ed03f42 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -88,7 +88,7 @@ impl LockAttempt {
 
     pub fn clone_for_test(&self) -> Self {
         Self {
-            target: self.target.clone,,
+            target: self.target.clone(),
             status: LockStatus::Succeded,
             requested_usage: self.requested_usage,
             heaviest_uncontended: Default::default(),

From 2f7ed313f068421fd1d4d2bd14280b3786e9c216 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 23 Aug 2022 12:36:10 +0900
Subject: [PATCH 1045/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index ec231b7ed03f42..e3f3bdbdfa0ade 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -481,7 +481,7 @@ impl Task {
         Self {
             unique_weight: self.unique_weight,
             tx: self.tx.clone(),
-            contention_count: Default:default()
+            contention_count: Default:default(),
             uncontended: Default::default(),
         }
     }

From d4424831cf626aee9542cc798fd67036560b18d3 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 23 Aug 2022 12:37:02 +0900
Subject: [PATCH 1046/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index e3f3bdbdfa0ade..5983bd441053fe 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -480,7 +480,7 @@ impl Task {
     pub fn clone_for_test(&self) {
         Self {
             unique_weight: self.unique_weight,
-            tx: self.tx.clone(),
+            tx: Box::new(self.tx.0.clone(), self.tx.1.clone_for_test()),
             contention_count: Default:default(),
             uncontended: Default::default(),
         }

From 5ea12264d9992e72d172961f25879c57a6d6a1a1 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 23 Aug 2022 12:37:34 +0900
Subject: [PATCH 1047/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 5983bd441053fe..27cb2d2219ce60 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -480,7 +480,7 @@ impl Task {
     pub fn clone_for_test(&self) {
         Self {
             unique_weight: self.unique_weight,
-            tx: Box::new(self.tx.0.clone(), self.tx.1.clone_for_test()),
+            tx: Box::new(self.tx.0.clone(), self.tx.1.iter(|l| l.clone_for_test()).collect::<Vec<_>>()),
             contention_count: Default:default(),
             uncontended: Default::default(),
         }

From 609702ee77089a4c4b7d393fcb0c96a221638edf Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 23 Aug 2022 12:37:57 +0900
Subject: [PATCH 1048/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 27cb2d2219ce60..d542c42fbb8e1e 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -480,7 +480,7 @@ impl Task {
     pub fn clone_for_test(&self) {
         Self {
             unique_weight: self.unique_weight,
-            tx: Box::new(self.tx.0.clone(), self.tx.1.iter(|l| l.clone_for_test()).collect::<Vec<_>>()),
+            tx: Box::new(self.tx.0.clone(), self.tx.1.iter().map(|l| l.clone_for_test()).collect::<Vec<_>>()),
             contention_count: Default:default(),
             uncontended: Default::default(),
         }

From 9f9cf6c99acb5242cb9ab2faac0e26e6cee9b82d Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 23 Aug 2022 12:38:07 +0900
Subject: [PATCH 1049/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index d542c42fbb8e1e..62ce2080324b21 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -480,7 +480,7 @@ impl Task {
     pub fn clone_for_test(&self) {
         Self {
             unique_weight: self.unique_weight,
-            tx: Box::new(self.tx.0.clone(), self.tx.1.iter().map(|l| l.clone_for_test()).collect::<Vec<_>>()),
+            tx: Box::new((self.tx.0.clone(), self.tx.1.iter().map(|l| l.clone_for_test()).collect::<Vec<_>>())),
             contention_count: Default:default(),
             uncontended: Default::default(),
         }

From 849ce1f6234a418910e8a364dbcb61e693b50567 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 23 Aug 2022 12:38:18 +0900
Subject: [PATCH 1050/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 62ce2080324b21..9ff0540bcf3680 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -481,7 +481,7 @@ impl Task {
         Self {
             unique_weight: self.unique_weight,
             tx: Box::new((self.tx.0.clone(), self.tx.1.iter().map(|l| l.clone_for_test()).collect::<Vec<_>>())),
-            contention_count: Default:default(),
+            contention_count: Default::default(),
             uncontended: Default::default(),
         }
     }

From 648cbdcb0628bef62562d8ac2171ae788de731b0 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 23 Aug 2022 12:38:33 +0900
Subject: [PATCH 1051/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 9ff0540bcf3680..b63dd7027e867f 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -477,7 +477,7 @@ impl Task {
         TaskInQueue::new(Self { unique_weight, tx, contention_count: 0, uncontended: Default::default() })
     }
 
-    pub fn clone_for_test(&self) {
+    pub fn clone_for_test(&self) -> Self {
         Self {
             unique_weight: self.unique_weight,
             tx: Box::new((self.tx.0.clone(), self.tx.1.iter().map(|l| l.clone_for_test()).collect::<Vec<_>>())),

From 8ca953b1aab44fe7b592674b1b5ffe0062ae622c Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 23 Aug 2022 12:39:19 +0900
Subject: [PATCH 1052/3199] save

---
 ledger-tool/src/main.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ledger-tool/src/main.rs b/ledger-tool/src/main.rs
index 0b014363bab0d2..3f71c55d2704b0 100644
--- a/ledger-tool/src/main.rs
+++ b/ledger-tool/src/main.rs
@@ -399,7 +399,7 @@ fn output_slot(
         let mut weight = 10_000_000;
         for i in 0..100 {
             error!("started!: {} {}", i, txes.len());
-            for tx in txes.clone() {
+            for tx in txes.iter().map(|t| t.clone_for_test()) {
                 while depth.load(Ordering::Relaxed) > 10_000 {
                     std::thread::sleep(std::time::Duration::from_micros(10));
                 }

From d853dce10c5e43921412719c456b48a566ea9984 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 23 Aug 2022 12:39:48 +0900
Subject: [PATCH 1053/3199] save

---
 ledger-tool/src/main.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ledger-tool/src/main.rs b/ledger-tool/src/main.rs
index 3f71c55d2704b0..17450c814469ea 100644
--- a/ledger-tool/src/main.rs
+++ b/ledger-tool/src/main.rs
@@ -399,7 +399,7 @@ fn output_slot(
         let mut weight = 10_000_000;
         for i in 0..100 {
             error!("started!: {} {}", i, txes.len());
-            for tx in txes.iter().map(|t| t.clone_for_test()) {
+            for tx in txes.iter().map(|t| Box::new((t.0.clone(), t.1.clone_for_test())) {
                 while depth.load(Ordering::Relaxed) > 10_000 {
                     std::thread::sleep(std::time::Duration::from_micros(10));
                 }

From 05044a53457b61c59f598a222c60d5290cf93745 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 23 Aug 2022 12:39:58 +0900
Subject: [PATCH 1054/3199] save

---
 ledger-tool/src/main.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ledger-tool/src/main.rs b/ledger-tool/src/main.rs
index 17450c814469ea..95af9f6e537eb1 100644
--- a/ledger-tool/src/main.rs
+++ b/ledger-tool/src/main.rs
@@ -399,7 +399,7 @@ fn output_slot(
         let mut weight = 10_000_000;
         for i in 0..100 {
             error!("started!: {} {}", i, txes.len());
-            for tx in txes.iter().map(|t| Box::new((t.0.clone(), t.1.clone_for_test())) {
+            for tx in txes.iter().map(|t| Box::new((t.0.clone(), t.1.clone_for_test()))) {
                 while depth.load(Ordering::Relaxed) > 10_000 {
                     std::thread::sleep(std::time::Duration::from_micros(10));
                 }

From 2d778486ba6f66c6942259b225ede425a822d2ce Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 23 Aug 2022 12:40:19 +0900
Subject: [PATCH 1055/3199] save

---
 ledger-tool/src/main.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ledger-tool/src/main.rs b/ledger-tool/src/main.rs
index 95af9f6e537eb1..a836fb51e8e60e 100644
--- a/ledger-tool/src/main.rs
+++ b/ledger-tool/src/main.rs
@@ -399,7 +399,7 @@ fn output_slot(
         let mut weight = 10_000_000;
         for i in 0..100 {
             error!("started!: {} {}", i, txes.len());
-            for tx in txes.iter().map(|t| Box::new((t.0.clone(), t.1.clone_for_test()))) {
+            for tx in txes.iter().map(|t| Box::new((t.0.clone(), t.1.iter().map(|l| l.clone_for_test())))) {
                 while depth.load(Ordering::Relaxed) > 10_000 {
                     std::thread::sleep(std::time::Duration::from_micros(10));
                 }

From da7aed297d13f15bc36f0b09fc60689d25d9e818 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 23 Aug 2022 12:40:39 +0900
Subject: [PATCH 1056/3199] save

---
 ledger-tool/src/main.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ledger-tool/src/main.rs b/ledger-tool/src/main.rs
index a836fb51e8e60e..0052933c3e29d7 100644
--- a/ledger-tool/src/main.rs
+++ b/ledger-tool/src/main.rs
@@ -399,7 +399,7 @@ fn output_slot(
         let mut weight = 10_000_000;
         for i in 0..100 {
             error!("started!: {} {}", i, txes.len());
-            for tx in txes.iter().map(|t| Box::new((t.0.clone(), t.1.iter().map(|l| l.clone_for_test())))) {
+            for tx in txes.iter().map(|t| Box::new((t.0.clone(), t.1.iter().map(|l| l.clone_for_test()).collect::<Vec<_>>()))) {
                 while depth.load(Ordering::Relaxed) > 10_000 {
                     std::thread::sleep(std::time::Duration::from_micros(10));
                 }

From 8e6dee2ea1de1d69d8f8d426b48f734720fe6c6d Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 23 Aug 2022 12:41:00 +0900
Subject: [PATCH 1057/3199] save

---
 ledger-tool/src/main.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ledger-tool/src/main.rs b/ledger-tool/src/main.rs
index 0052933c3e29d7..63600ed2208235 100644
--- a/ledger-tool/src/main.rs
+++ b/ledger-tool/src/main.rs
@@ -405,7 +405,7 @@ fn output_slot(
                 }
 
                 let t = solana_scheduler::Task::new_for_queue(weight, tx);
-                for lock_attempt in tx.1.iter() {
+                for lock_attempt in t.1.iter() {
                     lock_attempt.target.page_ref().contended_unique_weights.insert_task_id(weight, solana_scheduler::TaskInQueue::clone(&t));
                 }
 

From 32e2da3b0f2cd5c4dd10e481dddb7809177786e0 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 23 Aug 2022 12:41:06 +0900
Subject: [PATCH 1058/3199] save

---
 ledger-tool/src/main.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ledger-tool/src/main.rs b/ledger-tool/src/main.rs
index 63600ed2208235..b278a3b4f98c29 100644
--- a/ledger-tool/src/main.rs
+++ b/ledger-tool/src/main.rs
@@ -405,7 +405,7 @@ fn output_slot(
                 }
 
                 let t = solana_scheduler::Task::new_for_queue(weight, tx);
-                for lock_attempt in t.1.iter() {
+                for lock_attempt in t.tx.1.iter() {
                     lock_attempt.target.page_ref().contended_unique_weights.insert_task_id(weight, solana_scheduler::TaskInQueue::clone(&t));
                 }
 

From 24d439e7844ea0e8585f4158414d3fc69b230498 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 23 Aug 2022 12:46:08 +0900
Subject: [PATCH 1059/3199] save

---
 ledger-tool/src/main.rs | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/ledger-tool/src/main.rs b/ledger-tool/src/main.rs
index b278a3b4f98c29..c98bbe82410174 100644
--- a/ledger-tool/src/main.rs
+++ b/ledger-tool/src/main.rs
@@ -397,7 +397,12 @@ fn output_slot(
         }
 
         let mut weight = 10_000_000;
-        for i in 0..100 {
+        let loop_count = if std::env::var("INFINITE").is_ok() {
+            100000
+        } else {
+            100
+        }
+        for i in 0..loop_count {
             error!("started!: {} {}", i, txes.len());
             for tx in txes.iter().map(|t| Box::new((t.0.clone(), t.1.iter().map(|l| l.clone_for_test()).collect::<Vec<_>>()))) {
                 while depth.load(Ordering::Relaxed) > 10_000 {

From c469f3d4c40e6bf101e3446b45f079a121243a06 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 23 Aug 2022 12:46:14 +0900
Subject: [PATCH 1060/3199] save

---
 ledger-tool/src/main.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ledger-tool/src/main.rs b/ledger-tool/src/main.rs
index c98bbe82410174..db5d0343fdbc44 100644
--- a/ledger-tool/src/main.rs
+++ b/ledger-tool/src/main.rs
@@ -401,7 +401,7 @@ fn output_slot(
             100000
         } else {
             100
-        }
+        };
         for i in 0..loop_count {
             error!("started!: {} {}", i, txes.len());
             for tx in txes.iter().map(|t| Box::new((t.0.clone(), t.1.iter().map(|l| l.clone_for_test()).collect::<Vec<_>>()))) {

From 2efe3d8a5a6d6166e675416c05e7fc08a052a55c Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 23 Aug 2022 12:52:08 +0900
Subject: [PATCH 1061/3199] save

---
 ledger-tool/src/main.rs | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/ledger-tool/src/main.rs b/ledger-tool/src/main.rs
index db5d0343fdbc44..d887460c0acd0d 100644
--- a/ledger-tool/src/main.rs
+++ b/ledger-tool/src/main.rs
@@ -292,7 +292,8 @@ fn output_slot(
             //let muxed_sender = muxed_sender.clone();
 
             let t2 = std::thread::Builder::new()
-                .name(format!("blockstore_processor_{}", thx))
+                //.name(format!("blockstore_processor_{}", thx))
+                .name(format!("sol-lane{}", thx))
                 .spawn(move || {
                     use solana_metrics::datapoint_info;
                     let current_thread_name = std::thread::current().name().unwrap().to_string();

From 24907c305efcb58f7ac47fedde25565196fde422 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 23 Aug 2022 12:54:17 +0900
Subject: [PATCH 1062/3199] save

---
 scheduler/src/lib.rs | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index b63dd7027e867f..4aa2d2e5b05b49 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -55,7 +55,8 @@ impl PageRc {
     }
 
     pub fn page_ref(&self) -> &Page {
-        <MyRcInner as std::borrow::Borrow<_>>::borrow(&self.0)
+        //<MyRcInner as std::borrow::Borrow<_>>::borrow(&self.0)
+        *self
     }
 }
 

From 1bab9758664506a3113cebdb4c15f1e31b5ad299 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 23 Aug 2022 12:54:36 +0900
Subject: [PATCH 1063/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 4aa2d2e5b05b49..6c38d7daa4e227 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -56,7 +56,7 @@ impl PageRc {
 
     pub fn page_ref(&self) -> &Page {
         //<MyRcInner as std::borrow::Borrow<_>>::borrow(&self.0)
-        *self
+        self.as_ref()
     }
 }
 

From a9393ccd8c855246aa3c796afdc88c30569a71b3 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 23 Aug 2022 12:55:15 +0900
Subject: [PATCH 1064/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 6c38d7daa4e227..0ba0a0c1052296 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -56,7 +56,7 @@ impl PageRc {
 
     pub fn page_ref(&self) -> &Page {
         //<MyRcInner as std::borrow::Borrow<_>>::borrow(&self.0)
-        self.as_ref()
+        *self.0
     }
 }
 

From 34658a936b1ed6439a7248db638ed2ff6887ede2 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 23 Aug 2022 12:55:24 +0900
Subject: [PATCH 1065/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 0ba0a0c1052296..5ce3bc1933b388 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -56,7 +56,7 @@ impl PageRc {
 
     pub fn page_ref(&self) -> &Page {
         //<MyRcInner as std::borrow::Borrow<_>>::borrow(&self.0)
-        *self.0
+        *(self.0)
     }
 }
 

From 8ff75f44a16caa2627191245d3bce6e48947e979 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 23 Aug 2022 12:55:30 +0900
Subject: [PATCH 1066/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 5ce3bc1933b388..77e3236e7a10ba 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -56,7 +56,7 @@ impl PageRc {
 
     pub fn page_ref(&self) -> &Page {
         //<MyRcInner as std::borrow::Borrow<_>>::borrow(&self.0)
-        *(self.0)
+        &*(self.0)
     }
 }
 

From b75f61b02fc43b751d8a49a5ff6fb3f7e143cd8c Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 23 Aug 2022 13:09:29 +0900
Subject: [PATCH 1067/3199] save

---
 scheduler/src/lib.rs | 88 +-------------------------------------------
 1 file changed, 1 insertion(+), 87 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 77e3236e7a10ba..4e03fa2fe2cca2 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -131,36 +131,18 @@ pub enum RequestedUsage {
 
 #[derive(Debug, Default)]
 pub struct TaskIds {
-    //task_ids: std::collections::BTreeSet<UniqueWeight>,
     task_ids: crossbeam_skiplist::SkipMap<UniqueWeight, TaskInQueue>,
-    //cached_heaviest: Option<UniqueWeight>,
 }
 
 impl TaskIds {
     #[inline(never)]
     pub fn insert_task_id(&self, u: TaskId, task: TaskInQueue) {
-        /*
-        match self.cached_heaviest {
-            Some(c) if u > c => { self.cached_heaviest = Some(u) },
-            None => { self.cached_heaviest = Some(u); }
-            _ => {},
-        }
-        */
-             
         self.task_ids.insert(u, task);
     }
 
     #[inline(never)]
     pub fn remove_task_id(&self, u: &TaskId) {
-        let a = self.task_ids.remove(u);
-        /*
-        match self.cached_heaviest {
-            //Some(ref c) if u == c => { self.cached_heaviest = self.task_ids.last().copied() },
-            Some(ref c) if u == c => { self.cached_heaviest = self.task_ids.back().map(|e| *(e.value())) },
-            _ => {},
-        }
-        */
-        a;
+        self.task_ids.remove(u);
     }
 
     #[inline(never)]
@@ -170,16 +152,12 @@ impl TaskIds {
 
     #[inline(never)]
     fn heaviest_task_id(&self) -> Option<TaskId> {
-        //self.task_ids.last()
         self.task_ids.back().map(|e| *(e.key()))
-        //self.cached_heaviest.as_ref()
     }
 
     #[inline(never)]
     pub fn heaviest_task_cursor(&self) -> Option<crossbeam_skiplist::map::Entry<'_, UniqueWeight, TaskInQueue>> {
-        //self.task_ids.last()
         self.task_ids.back()
-        //self.cached_heaviest.as_ref()
     }
 }
 
@@ -285,8 +263,6 @@ impl AddressBook {
                         }
                         RequestedUsage::Writable => {
                             if from_runnable || prefer_immediate {
-                                //Self::remember_address_contention(&mut page, unique_weight);
-                                //*remembered = true;
                                 *status = LockStatus::Failed;
                             } else {
                                 match page.next_usage {
@@ -299,15 +275,11 @@ impl AddressBook {
                                         *status = LockStatus::Failed;
                                     },
                                 }
-                                
-                                //*status = LockStatus::Failed;
                             }
                         }
                     },
                     Usage::Writable => {
                         if from_runnable || prefer_immediate {
-                            //Self::remember_address_contention(&mut page, unique_weight);
-                            //*remembered = true;
                             *status = LockStatus::Failed;
                         } else {
                             match page.next_usage {
@@ -320,26 +292,11 @@ impl AddressBook {
                                     *status = LockStatus::Failed;
                                 },
                             }
-                            //*status = LockStatus::Failed;
                         }
                     },
                 }
     }
 
-    /*
-    #[inline(never)]
-    fn remember_address_contention(page: &mut Page, unique_weight: &UniqueWeight) {
-        page.contended_unique_weights.insert_task_id(*unique_weight);
-    }
-
-    #[inline(never)]
-    fn forget_address_contention(unique_weight: &UniqueWeight, a: &mut LockAttempt) {
-        if a.remembered {
-            a.target.page_ref().contended_unique_weights.remove_task_id(unique_weight);
-        }
-    }
-    */
-
     fn reset_lock(&mut self, attempt: &mut LockAttempt, after_execution: bool) -> bool {
         match attempt.status {
             LockStatus::Succeded => {
@@ -653,21 +610,6 @@ impl ScheduleStage {
         *unique_key -= 1;
     }
 
-    /*
-    #[inline(never)]
-    fn get_newly_u_u_w<'a>(
-        address: &'a Pubkey,
-        address_book: &'a AddressBook,
-    ) -> &'a std::collections::BTreeSet<UniqueWeight> {
-        &address_book
-            .book
-            .get(address)
-            .unwrap()
-            .0
-            .contended_unique_weights
-    }
-    */
-
     #[inline(never)]
     fn get_heaviest_from_contended<'a>(address_book: &'a mut AddressBook) -> Option<std::collections::btree_map::OccupiedEntry<'a, UniqueWeight, TaskInQueue>> {
         address_book.uncontended_task_ids.last_entry()
@@ -806,11 +748,6 @@ impl ScheduleStage {
             }
 
             trace!("successful lock: (from_runnable: {}) after {} contentions", from_runnable, next_task.contention_count);
-            Self::finalize_lock_before_execution(
-                address_book,
-                &unique_weight,
-                &mut next_task.tx.1,
-            );
             return Some((unique_weight, arc_next_task));
         } else {
             break;
@@ -820,19 +757,6 @@ impl ScheduleStage {
         None
     }
 
-    #[inline(never)]
-    // naming: relock_before_execution() / update_address_book() / update_uncontended_addresses()?
-    fn finalize_lock_before_execution(
-        address_book: &mut AddressBook,
-        unique_weight: &UniqueWeight,
-        lock_attempts: &mut Vec<LockAttempt>,
-    ) {
-        for mut l in lock_attempts {
-            // ensure to remove remaining refs of this unique_weight
-            //AddressBook::forget_address_contention(&unique_weight, &mut l);
-        }
-    }
-
     #[inline(never)]
     fn finalize_lock_for_provisional_execution(
         address_book: &mut AddressBook,
@@ -841,7 +765,6 @@ impl ScheduleStage {
         provisional_count: usize,
     ) {
         for mut l in lock_attempts {
-            //AddressBook::forget_address_contention(&unique_weight, &mut l);
             match l.status {
                 LockStatus::Provisional => {
                     l.target.page_mut().provisional_task_ids.insert(*unique_weight);
@@ -865,16 +788,7 @@ impl ScheduleStage {
         from_runnable: bool,
     ) {
         for l in lock_attempts {
-            if from_runnable {
-                //AddressBook::remember_address_contention(&mut l.target.page_mut(), unique_weight);
-                //l.remembered = true;
-            }
             address_book.reset_lock(l, false);
-            //if let Some(uw) = l.target.page().contended_unique_weights.last() {
-            //    address_book.uncontended_task_ids.remove(uw);
-            //}
-
-            // todo: mem::forget and panic in LockAttempt::drop()
         }
     }
 

From 625ed7d3684fd641cdf55f94c20dff393703b232 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 23 Aug 2022 14:05:05 +0900
Subject: [PATCH 1068/3199] save

---
 scheduler/Cargo.toml | 6 ------
 1 file changed, 6 deletions(-)

diff --git a/scheduler/Cargo.toml b/scheduler/Cargo.toml
index b678e0644e8ef6..07852e7964f869 100644
--- a/scheduler/Cargo.toml
+++ b/scheduler/Cargo.toml
@@ -11,14 +11,10 @@ documentation = "https://docs.rs/solana-scheduler"
 publish = true
 
 [dependencies]
-atomic_enum = "0.1.1"
 arc-swap = "1.5.0"
-by_address = "1.0.4"
 crossbeam-channel = "0.5.5"
 crossbeam-skiplist = { path = "../../../crossbeam/crossbeam-skiplist" }
 dashmap = { version = "4.0.2" }
-rayon = "1.5.3"
-im = { version = "15.1.0", features = ["rayon", "serde"] }
 log = "0.4.17"
 solana-entry = { path = "../entry", version = "=1.11.5" }
 solana-runtime = { path = "../runtime", version = "=1.11.5" }
@@ -29,8 +25,6 @@ solana-version = { path = "../version", version = "=1.11.5" }
 solana-metrics = { path = "../metrics", version = "=1.11.5" }
 solana-measure = { path = "../measure", version = "=1.11.5" }
 sha2 = { version = "0.10.2" }
-rand = "0.7.0"
-
 
 [[bin]]
 name = "solana-scheduler"

From b797f5ca26bdbf31b98675fb731092e1e0891d8a Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 23 Aug 2022 14:05:32 +0900
Subject: [PATCH 1069/3199] save

---
 Cargo.lock           | 22 ----------------------
 scheduler/Cargo.toml |  1 +
 2 files changed, 1 insertion(+), 22 deletions(-)

diff --git a/Cargo.lock b/Cargo.lock
index 4592102ddee94a..718ffb25a38891 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -248,17 +248,6 @@ dependencies = [
  "syn 1.0.98",
 ]
 
-[[package]]
-name = "atomic_enum"
-version = "0.1.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6a57588a20357c74ee9036656b7fad5c54328ee8657c6ff29ba31fd5ea503a1c"
-dependencies = [
- "proc-macro2 1.0.41",
- "quote 1.0.18",
- "syn 1.0.98",
-]
-
 [[package]]
 name = "atty"
 version = "0.2.14"
@@ -581,12 +570,6 @@ dependencies = [
  "serde",
 ]
 
-[[package]]
-name = "by_address"
-version = "1.0.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e245704f60eb4eb45810d65cf14eb54d2eb50a6f3715fe2d7cd01ee905c2944f"
-
 [[package]]
 name = "byte-tools"
 version = "0.3.1"
@@ -6103,15 +6086,10 @@ name = "solana-scheduler"
 version = "1.11.5"
 dependencies = [
  "arc-swap",
- "atomic_enum",
- "by_address",
  "crossbeam-channel",
  "crossbeam-skiplist",
  "dashmap",
- "im",
  "log",
- "rand 0.7.3",
- "rayon",
  "sha2 0.10.2",
  "solana-entry",
  "solana-logger 1.11.5",
diff --git a/scheduler/Cargo.toml b/scheduler/Cargo.toml
index 07852e7964f869..daaff8ae54142e 100644
--- a/scheduler/Cargo.toml
+++ b/scheduler/Cargo.toml
@@ -25,6 +25,7 @@ solana-version = { path = "../version", version = "=1.11.5" }
 solana-metrics = { path = "../metrics", version = "=1.11.5" }
 solana-measure = { path = "../measure", version = "=1.11.5" }
 sha2 = { version = "0.10.2" }
+rand = "0.7.0"
 
 [[bin]]
 name = "solana-scheduler"

From 7d71429fc6e742215569d6382bdf613b47028b6e Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 23 Aug 2022 14:05:43 +0900
Subject: [PATCH 1070/3199] save

---
 Cargo.lock           | 1 +
 scheduler/src/lib.rs | 1 -
 2 files changed, 1 insertion(+), 1 deletion(-)

diff --git a/Cargo.lock b/Cargo.lock
index 718ffb25a38891..393a336c9ffa0d 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -6090,6 +6090,7 @@ dependencies = [
  "crossbeam-skiplist",
  "dashmap",
  "log",
+ "rand 0.7.3",
  "sha2 0.10.2",
  "solana-entry",
  "solana-logger 1.11.5",
diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 4e03fa2fe2cca2..9b4f0ff37a4b46 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -2,7 +2,6 @@
 #![feature(get_mut_unchecked)]
 
 use {
-    atomic_enum::atomic_enum,
     crossbeam_channel::{bounded, unbounded},
     log::*,
     rand::Rng,

From 6de987a35155f126318dfa95bfc56174c6b0d2f2 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 23 Aug 2022 14:38:00 +0900
Subject: [PATCH 1071/3199] save

---
 scheduler/src/lib.rs | 28 ----------------------------
 1 file changed, 28 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 9b4f0ff37a4b46..8f72f875055c08 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -483,39 +483,11 @@ impl TaskQueue {
         debug_assert!(pre_existed.is_none()); //, "identical shouldn't exist: {:?}", unique_weight);
     }
 
-    #[inline(never)]
-    fn entry_to_execute(
-        &mut self,
-        unique_weight: UniqueWeight,
-    ) -> Option<TaskQueueOccupiedEntry<'_>> {
-        let queue_entry = self.tasks.entry(unique_weight);
-        match queue_entry {
-            TaskQueueEntry::Occupied(queue_entry) => Some(queue_entry),
-            TaskQueueEntry::Vacant(_queue_entry) => None,
-        }
-    }
-
-    #[inline(never)]
-    pub fn has_contended_task(&self, unique_weight: &UniqueWeight) -> bool {
-        let maybe_task = self.tasks.get(unique_weight);
-        match maybe_task {
-            Some(task) => task.currently_contended(),
-            None => false,
-        }
-    }
-
     #[inline(never)]
     fn heaviest_entry_to_execute(
         &mut self,
     ) -> Option<TaskQueueOccupiedEntry<'_>> {
-        //panic!()
         self.tasks.last_entry()
-        //let k = self.tasks.get_max().map(|(k, _v)| *k);
-        //panic!();
-        //let k = self.tasks.iter().next().map(|(k, _v)| *k);
-        //let k = self.tasks.iter().next().map(|r| *r.key());
-
-        //k.map(|k| self.entry_to_execute(k).unwrap())
     }
 
     fn task_count(&self) -> usize {

From df3b113147c324a9500f342ae8ef4c1ff3aa8659 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 23 Aug 2022 15:04:55 +0900
Subject: [PATCH 1072/3199] save

---
 ledger-tool/src/main.rs | 52 ++++++++++++++++++++++++-----------------
 1 file changed, 31 insertions(+), 21 deletions(-)

diff --git a/ledger-tool/src/main.rs b/ledger-tool/src/main.rs
index d887460c0acd0d..5bfada1adda190 100644
--- a/ledger-tool/src/main.rs
+++ b/ledger-tool/src/main.rs
@@ -403,31 +403,41 @@ fn output_slot(
         } else {
             100
         };
-        for i in 0..loop_count {
-            error!("started!: {} {}", i, txes.len());
-            for tx in txes.iter().map(|t| Box::new((t.0.clone(), t.1.iter().map(|l| l.clone_for_test()).collect::<Vec<_>>()))) {
-                while depth.load(Ordering::Relaxed) > 10_000 {
-                    std::thread::sleep(std::time::Duration::from_micros(10));
-                }
+        let handles2 = (0..lane_count)
+            .map(|thx| {
+                let t1 = std::thread::Builder::new()
+                    .name("sol-producer{}".to_string())
+                    .spawn(move || loop {
+                        for i in 0..loop_count {
+                            error!("started!: {} {}", i, txes.len());
+                            for tx in txes.iter().map(|t| Box::new((t.0.clone(), t.1.iter().map(|l| l.clone_for_test()).collect::<Vec<_>>()))) {
+                                while depth.load(Ordering::Relaxed) > 10_000 {
+                                    std::thread::sleep(std::time::Duration::from_micros(10));
+                                }
 
-                let t = solana_scheduler::Task::new_for_queue(weight, tx);
-                for lock_attempt in t.tx.1.iter() {
-                    lock_attempt.target.page_ref().contended_unique_weights.insert_task_id(weight, solana_scheduler::TaskInQueue::clone(&t));
-                }
+                                let t = solana_scheduler::Task::new_for_queue(weight, tx);
+                                for lock_attempt in t.tx.1.iter() {
+                                    lock_attempt.target.page_ref().contended_unique_weights.insert_task_id(weight, solana_scheduler::TaskInQueue::clone(&t));
+                                }
+
+                                muxed_sender
+                                    .send(solana_scheduler::Multiplexed::FromPrevious((
+                                        //Weight { ix: weight },
+                                        weight,
+                                        t,
+                                    )))
+                                    .unwrap();
+                                depth.fetch_add(1, Ordering::Relaxed);
+                                weight -= 1;
+                            }
+                        }
+                    }).unwrap();
+                t1
+        }.collect::<Vec<_>>();
 
-                muxed_sender
-                    .send(solana_scheduler::Multiplexed::FromPrevious((
-                        //Weight { ix: weight },
-                        weight,
-                        t,
-                    )))
-                    .unwrap();
-                depth.fetch_add(1, Ordering::Relaxed);
-                weight -= 1;
-            }
-        }
         t1.join().unwrap();
         handles.into_iter().for_each(|t| t.join().unwrap());
+        handles2.into_iter().for_each(|t| t.join().unwrap());
         t3.join().unwrap();
 
         output_slot_rewards(blockstore, slot, method);

From f07c3de9c39cefb6f46c2f79695b830fcb2b250f Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 23 Aug 2022 15:05:11 +0900
Subject: [PATCH 1073/3199] save

---
 ledger-tool/src/main.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ledger-tool/src/main.rs b/ledger-tool/src/main.rs
index 5bfada1adda190..1a61a60624abbb 100644
--- a/ledger-tool/src/main.rs
+++ b/ledger-tool/src/main.rs
@@ -433,7 +433,7 @@ fn output_slot(
                         }
                     }).unwrap();
                 t1
-        }.collect::<Vec<_>>();
+        }).collect::<Vec<_>>();
 
         t1.join().unwrap();
         handles.into_iter().for_each(|t| t.join().unwrap());

From a12e1dad1a90a0d4e312ffdea51250d4d956d099 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 23 Aug 2022 15:05:53 +0900
Subject: [PATCH 1074/3199] save

---
 ledger-tool/src/main.rs | 1 +
 1 file changed, 1 insertion(+)

diff --git a/ledger-tool/src/main.rs b/ledger-tool/src/main.rs
index 1a61a60624abbb..63b8da5c118769 100644
--- a/ledger-tool/src/main.rs
+++ b/ledger-tool/src/main.rs
@@ -405,6 +405,7 @@ fn output_slot(
         };
         let handles2 = (0..lane_count)
             .map(|thx| {
+                let depth = depth.clone();
                 let t1 = std::thread::Builder::new()
                     .name("sol-producer{}".to_string())
                     .spawn(move || loop {

From 1c4cf32f3e47b59821c558e72f0e9a7d41e00e18 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 23 Aug 2022 15:06:32 +0900
Subject: [PATCH 1075/3199] save

---
 ledger-tool/src/main.rs | 1 +
 1 file changed, 1 insertion(+)

diff --git a/ledger-tool/src/main.rs b/ledger-tool/src/main.rs
index 63b8da5c118769..8a75e1a1bea833 100644
--- a/ledger-tool/src/main.rs
+++ b/ledger-tool/src/main.rs
@@ -396,6 +396,7 @@ fn output_slot(
                 skip_voting,
             );
         }
+        let txes = Arc::new(txes);
 
         let mut weight = 10_000_000;
         let loop_count = if std::env::var("INFINITE").is_ok() {

From 7aa3b7657ff1cda0e14f369e70841ac2cb9917dc Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 23 Aug 2022 15:06:44 +0900
Subject: [PATCH 1076/3199] save

---
 ledger-tool/src/main.rs | 1 +
 1 file changed, 1 insertion(+)

diff --git a/ledger-tool/src/main.rs b/ledger-tool/src/main.rs
index 8a75e1a1bea833..a53de957990573 100644
--- a/ledger-tool/src/main.rs
+++ b/ledger-tool/src/main.rs
@@ -407,6 +407,7 @@ fn output_slot(
         let handles2 = (0..lane_count)
             .map(|thx| {
                 let depth = depth.clone();
+                let txes = txes.clone();
                 let t1 = std::thread::Builder::new()
                     .name("sol-producer{}".to_string())
                     .spawn(move || loop {

From 443873fbc9e1a14b9d137697a871ac3dc749d02e Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 23 Aug 2022 15:07:10 +0900
Subject: [PATCH 1077/3199] save

---
 ledger-tool/src/main.rs | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/ledger-tool/src/main.rs b/ledger-tool/src/main.rs
index a53de957990573..6e74bd6bbd5c03 100644
--- a/ledger-tool/src/main.rs
+++ b/ledger-tool/src/main.rs
@@ -404,10 +404,11 @@ fn output_slot(
         } else {
             100
         };
-        let handles2 = (0..lane_count)
+        let handles2 = (0..4)
             .map(|thx| {
                 let depth = depth.clone();
                 let txes = txes.clone();
+                let muxed_sender = muxed_sender.clone();
                 let t1 = std::thread::Builder::new()
                     .name("sol-producer{}".to_string())
                     .spawn(move || loop {

From b6bf92c1e566432010f5f4b9bdb7a33eaa3d8f4b Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 23 Aug 2022 15:08:43 +0900
Subject: [PATCH 1078/3199] save

---
 ledger-tool/src/main.rs | 37 ++++++++++++++++++++-----------------
 1 file changed, 20 insertions(+), 17 deletions(-)

diff --git a/ledger-tool/src/main.rs b/ledger-tool/src/main.rs
index 6e74bd6bbd5c03..a61c2b9b2ab506 100644
--- a/ledger-tool/src/main.rs
+++ b/ledger-tool/src/main.rs
@@ -363,23 +363,26 @@ fn output_slot(
     let depth = Arc::new(std::sync::atomic::AtomicUsize::default());
 
     let d = depth.clone();
-    let t3 = std::thread::Builder::new()
-        .name("sol-consumer".to_string())
-        .spawn(move || {
-            for step in 0.. {
-                let ee = post_schedule_env_receiver.recv().unwrap();
-                d.fetch_sub(1, Ordering::Relaxed);
-                trace!(
-                    "post schedule stage: #{} {:#?}",
-                    step,
-                    ee.task.tx.0.signature()
-                );
-                if step % 1966 == 0 {
-                    error!("finished!: {} {}", step, post_schedule_env_receiver.len());
+    let handles3 = (0..4).map(|thx| {
+        let t3 = std::thread::Builder::new()
+            .name("sol-consumer".to_string())
+            .spawn(move || {
+                for step in 0.. {
+                    let ee = post_schedule_env_receiver.recv().unwrap();
+                    d.fetch_sub(1, Ordering::Relaxed);
+                    trace!(
+                        "post schedule stage: #{} {:#?}",
+                        step,
+                        ee.task.tx.0.signature()
+                    );
+                    if step % 1966 == 0 {
+                        error!("finished!: {} {}", step, post_schedule_env_receiver.len());
+                    }
                 }
-            }
-        })
-        .unwrap();
+            })
+            .unwrap();
+            t3
+    }).collect::<Vec<_>>();
 
     if verbose_level >= 2 {
         let mut txes = Vec::new();
@@ -442,7 +445,7 @@ fn output_slot(
         t1.join().unwrap();
         handles.into_iter().for_each(|t| t.join().unwrap());
         handles2.into_iter().for_each(|t| t.join().unwrap());
-        t3.join().unwrap();
+        handles3.into_iter().for_each(|t| t.join().unwrap());
 
         output_slot_rewards(blockstore, slot, method);
     } else if verbose_level >= 1 {

From a35c643c8a42fb775318d761740314d1a015d386 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 23 Aug 2022 15:09:14 +0900
Subject: [PATCH 1079/3199] save

---
 ledger-tool/src/main.rs | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/ledger-tool/src/main.rs b/ledger-tool/src/main.rs
index a61c2b9b2ab506..a9e988d8f60ea7 100644
--- a/ledger-tool/src/main.rs
+++ b/ledger-tool/src/main.rs
@@ -365,8 +365,9 @@ fn output_slot(
     let d = depth.clone();
     let handles3 = (0..4).map(|thx| {
         let t3 = std::thread::Builder::new()
-            .name("sol-consumer".to_string())
+            .name("sol-consumer{}".to_string())
             .spawn(move || {
+                let post_schedule_env_receiver = post_schedule_env_receiver.clone();
                 for step in 0.. {
                     let ee = post_schedule_env_receiver.recv().unwrap();
                     d.fetch_sub(1, Ordering::Relaxed);

From 471a474a43e048e25a46de0697fab172e325db1d Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 23 Aug 2022 15:09:51 +0900
Subject: [PATCH 1080/3199] save

---
 ledger-tool/src/main.rs | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/ledger-tool/src/main.rs b/ledger-tool/src/main.rs
index a9e988d8f60ea7..5361dc8545ade9 100644
--- a/ledger-tool/src/main.rs
+++ b/ledger-tool/src/main.rs
@@ -364,10 +364,11 @@ fn output_slot(
 
     let d = depth.clone();
     let handles3 = (0..4).map(|thx| {
+        let post_schedule_env_receiver = post_schedule_env_receiver.clone();
+
         let t3 = std::thread::Builder::new()
             .name("sol-consumer{}".to_string())
             .spawn(move || {
-                let post_schedule_env_receiver = post_schedule_env_receiver.clone();
                 for step in 0.. {
                     let ee = post_schedule_env_receiver.recv().unwrap();
                     d.fetch_sub(1, Ordering::Relaxed);

From 4763cf91c359a7b3fbc8b30831119cd06dca2ed5 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 23 Aug 2022 15:10:16 +0900
Subject: [PATCH 1081/3199] save

---
 ledger-tool/src/main.rs | 1 +
 1 file changed, 1 insertion(+)

diff --git a/ledger-tool/src/main.rs b/ledger-tool/src/main.rs
index 5361dc8545ade9..e2848403327f45 100644
--- a/ledger-tool/src/main.rs
+++ b/ledger-tool/src/main.rs
@@ -365,6 +365,7 @@ fn output_slot(
     let d = depth.clone();
     let handles3 = (0..4).map(|thx| {
         let post_schedule_env_receiver = post_schedule_env_receiver.clone();
+        let d = d.clone();
 
         let t3 = std::thread::Builder::new()
             .name("sol-consumer{}".to_string())

From aea84dcb8bf9fa880729a0f949c07a180f200497 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 23 Aug 2022 15:12:47 +0900
Subject: [PATCH 1082/3199] save

---
 ledger-tool/src/main.rs | 16 +++++++++++++---
 1 file changed, 13 insertions(+), 3 deletions(-)

diff --git a/ledger-tool/src/main.rs b/ledger-tool/src/main.rs
index e2848403327f45..78771b75992574 100644
--- a/ledger-tool/src/main.rs
+++ b/ledger-tool/src/main.rs
@@ -363,14 +363,20 @@ fn output_slot(
     let depth = Arc::new(std::sync::atomic::AtomicUsize::default());
 
     let d = depth.clone();
-    let handles3 = (0..4).map(|thx| {
+    let consumer_count = std::env::var("CONSUMER_COUNT")
+        .unwrap_or(format!("{}", std::thread::available_parallelism().unwrap()))
+        .parse::<usize>()
+        .unwrap();
+    let step = Arc::new(std::sync::atomic::AtomicUsize::default());
+    let handles3 = (0..consumer_count).map(|thx| {
         let post_schedule_env_receiver = post_schedule_env_receiver.clone();
         let d = d.clone();
+        let step = step.clone();
 
         let t3 = std::thread::Builder::new()
             .name("sol-consumer{}".to_string())
             .spawn(move || {
-                for step in 0.. {
+                loop {
                     let ee = post_schedule_env_receiver.recv().unwrap();
                     d.fetch_sub(1, Ordering::Relaxed);
                     trace!(
@@ -410,7 +416,11 @@ fn output_slot(
         } else {
             100
         };
-        let handles2 = (0..4)
+        let producer_count = std::env::var("PRODUCER_COUNT")
+            .unwrap_or(format!("{}", std::thread::available_parallelism().unwrap()))
+            .parse::<usize>()
+            .unwrap();
+        let handles2 = (0..producer_count)
             .map(|thx| {
                 let depth = depth.clone();
                 let txes = txes.clone();

From 3f20870df139728db29d774d1a052a3236a63b2c Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 23 Aug 2022 15:13:11 +0900
Subject: [PATCH 1083/3199] save

---
 ledger-tool/src/main.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/ledger-tool/src/main.rs b/ledger-tool/src/main.rs
index 78771b75992574..630ef1ff9045b4 100644
--- a/ledger-tool/src/main.rs
+++ b/ledger-tool/src/main.rs
@@ -364,7 +364,7 @@ fn output_slot(
 
     let d = depth.clone();
     let consumer_count = std::env::var("CONSUMER_COUNT")
-        .unwrap_or(format!("{}", std::thread::available_parallelism().unwrap()))
+        .unwrap_or(format!("{}", 4))
         .parse::<usize>()
         .unwrap();
     let step = Arc::new(std::sync::atomic::AtomicUsize::default());
@@ -417,7 +417,7 @@ fn output_slot(
             100
         };
         let producer_count = std::env::var("PRODUCER_COUNT")
-            .unwrap_or(format!("{}", std::thread::available_parallelism().unwrap()))
+            .unwrap_or(format!("{}", 4))
             .parse::<usize>()
             .unwrap();
         let handles2 = (0..producer_count)

From c777864a88b9d7ea6fb9d0746a90414ffad2df2a Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 23 Aug 2022 15:13:49 +0900
Subject: [PATCH 1084/3199] save

---
 ledger-tool/src/main.rs | 1 +
 1 file changed, 1 insertion(+)

diff --git a/ledger-tool/src/main.rs b/ledger-tool/src/main.rs
index 630ef1ff9045b4..5df201a531ac49 100644
--- a/ledger-tool/src/main.rs
+++ b/ledger-tool/src/main.rs
@@ -379,6 +379,7 @@ fn output_slot(
                 loop {
                     let ee = post_schedule_env_receiver.recv().unwrap();
                     d.fetch_sub(1, Ordering::Relaxed);
+                    let step = step.fetch_add(1, Ordering::Relaxed);
                     trace!(
                         "post schedule stage: #{} {:#?}",
                         step,

From b981b539366f776c3957dfaeacc958762c8f16db Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 23 Aug 2022 15:18:13 +0900
Subject: [PATCH 1085/3199] save

---
 ledger-tool/src/main.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/ledger-tool/src/main.rs b/ledger-tool/src/main.rs
index 5df201a531ac49..5c2da734603404 100644
--- a/ledger-tool/src/main.rs
+++ b/ledger-tool/src/main.rs
@@ -374,7 +374,7 @@ fn output_slot(
         let step = step.clone();
 
         let t3 = std::thread::Builder::new()
-            .name("sol-consumer{}".to_string())
+            .name(format!("sol-consumer{}", thx))
             .spawn(move || {
                 loop {
                     let ee = post_schedule_env_receiver.recv().unwrap();
@@ -427,7 +427,7 @@ fn output_slot(
                 let txes = txes.clone();
                 let muxed_sender = muxed_sender.clone();
                 let t1 = std::thread::Builder::new()
-                    .name("sol-producer{}".to_string())
+                    .name(format!("sol-producer{}", thx))
                     .spawn(move || loop {
                         for i in 0..loop_count {
                             error!("started!: {} {}", i, txes.len());

From f5655010641f177b4adaad6573ce01e5314279b4 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 23 Aug 2022 15:22:02 +0900
Subject: [PATCH 1086/3199] save

---
 transaction-scheduler-bench/src/main.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/transaction-scheduler-bench/src/main.rs b/transaction-scheduler-bench/src/main.rs
index 9505c2964236e6..cf7fc5bd897bb1 100644
--- a/transaction-scheduler-bench/src/main.rs
+++ b/transaction-scheduler-bench/src/main.rs
@@ -343,7 +343,7 @@ fn handle_transaction_batch(
         .fetch_add(priority_collected, Ordering::Relaxed);
 
     let uq = transaction_batch.unique_weight;
-    for lock_attempt in transaction_batch.task.tx.1.iter() {
+    for lock_attempt in transaction_batch.task.tx.1.iter_mut() {
         let page = lock_attempt.target.page_ref();
         page.contended_unique_weights.remove_task_id(&uq);
         if let Some(mut task_cursor) = page.contended_unique_weights.heaviest_task_cursor() {

From d0c84e63fc1c80656856577f1ff0205f8ad047ff Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 23 Aug 2022 15:22:38 +0900
Subject: [PATCH 1087/3199] save

---
 ledger-tool/src/main.rs                 | 2 +-
 transaction-scheduler-bench/src/main.rs | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/ledger-tool/src/main.rs b/ledger-tool/src/main.rs
index 5c2da734603404..de3e4c98decbdb 100644
--- a/ledger-tool/src/main.rs
+++ b/ledger-tool/src/main.rs
@@ -300,7 +300,7 @@ fn output_slot(
                     let send_metrics = std::env::var("SEND_METRICS").is_ok();
 
                     for step in 0.. {
-                        let ee = pre_execute_env_receiver.recv().unwrap();
+                        let mut ee = pre_execute_env_receiver.recv().unwrap();
                         if step % 1966 == 0 {
                             error!("executing!: {} {}", step, pre_execute_env_receiver.len());
                         }
diff --git a/transaction-scheduler-bench/src/main.rs b/transaction-scheduler-bench/src/main.rs
index cf7fc5bd897bb1..9505c2964236e6 100644
--- a/transaction-scheduler-bench/src/main.rs
+++ b/transaction-scheduler-bench/src/main.rs
@@ -343,7 +343,7 @@ fn handle_transaction_batch(
         .fetch_add(priority_collected, Ordering::Relaxed);
 
     let uq = transaction_batch.unique_weight;
-    for lock_attempt in transaction_batch.task.tx.1.iter_mut() {
+    for lock_attempt in transaction_batch.task.tx.1.iter() {
         let page = lock_attempt.target.page_ref();
         page.contended_unique_weights.remove_task_id(&uq);
         if let Some(mut task_cursor) = page.contended_unique_weights.heaviest_task_cursor() {

From 33b86c9744f905b9dfbd5cc58f0d4d04c98cc112 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 23 Aug 2022 15:23:12 +0900
Subject: [PATCH 1088/3199] save

---
 ledger-tool/src/main.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ledger-tool/src/main.rs b/ledger-tool/src/main.rs
index de3e4c98decbdb..320cdd573b9312 100644
--- a/ledger-tool/src/main.rs
+++ b/ledger-tool/src/main.rs
@@ -334,7 +334,7 @@ fn output_slot(
                             .unwrap();
                         */
                         let uq = ee.unique_weight;
-                        for lock_attempt in ee.task.tx.1.iter() {
+                        for mut lock_attempt in ee.task.tx.1.mut_iter() {
                             let page = lock_attempt.target.page_ref();
                             page.contended_unique_weights.remove_task_id(&uq);
                             if let Some(mut task_cursor) = page.contended_unique_weights.heaviest_task_cursor() {

From 4c604582a9f9277fa9288a50e885b0aff83c3a80 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 23 Aug 2022 15:23:20 +0900
Subject: [PATCH 1089/3199] save

---
 ledger-tool/src/main.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ledger-tool/src/main.rs b/ledger-tool/src/main.rs
index 320cdd573b9312..e70fd012dbb64c 100644
--- a/ledger-tool/src/main.rs
+++ b/ledger-tool/src/main.rs
@@ -334,7 +334,7 @@ fn output_slot(
                             .unwrap();
                         */
                         let uq = ee.unique_weight;
-                        for mut lock_attempt in ee.task.tx.1.mut_iter() {
+                        for mut lock_attempt in ee.task.tx.1.iter_mut() {
                             let page = lock_attempt.target.page_ref();
                             page.contended_unique_weights.remove_task_id(&uq);
                             if let Some(mut task_cursor) = page.contended_unique_weights.heaviest_task_cursor() {

From e9589b2a9e1a058effa046b82cfd456048961258 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 23 Aug 2022 15:23:55 +0900
Subject: [PATCH 1090/3199] save

---
 ledger-tool/src/main.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ledger-tool/src/main.rs b/ledger-tool/src/main.rs
index e70fd012dbb64c..5adfbb433e9d8d 100644
--- a/ledger-tool/src/main.rs
+++ b/ledger-tool/src/main.rs
@@ -334,7 +334,7 @@ fn output_slot(
                             .unwrap();
                         */
                         let uq = ee.unique_weight;
-                        for mut lock_attempt in ee.task.tx.1.iter_mut() {
+                        for mut lock_attempt in ee.lock_attempts.iter_mut() {
                             let page = lock_attempt.target.page_ref();
                             page.contended_unique_weights.remove_task_id(&uq);
                             if let Some(mut task_cursor) = page.contended_unique_weights.heaviest_task_cursor() {

From 42a5ce24c3d48220881bf46b6214e5708266a7b4 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 23 Aug 2022 15:24:51 +0900
Subject: [PATCH 1091/3199] save

---
 scheduler/src/lib.rs | 1 +
 1 file changed, 1 insertion(+)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 8f72f875055c08..e842b14f37ba49 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -33,6 +33,7 @@ pub struct ExecutionEnvironment {
     pub cu: usize,
     pub unique_weight: UniqueWeight,
     pub task: TaskInQueue,
+    pub lock_attempts: Vec<LockAttempt>,
 }
 
 impl ExecutionEnvironment {

From c20f6f2d073e73400dff6d17b82cdc03fd13ee5a Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 23 Aug 2022 15:26:01 +0900
Subject: [PATCH 1092/3199] save

---
 scheduler/src/lib.rs | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index e842b14f37ba49..8037659b7b848e 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -650,10 +650,11 @@ impl ScheduleStage {
         address_book: &mut AddressBook,
         contended_count: &mut usize,
         prefer_immediate: bool,
-    ) -> Option<(UniqueWeight, TaskInQueue)> {
+    ) -> Option<(UniqueWeight, TaskInQueue, Vec<LockAttempt>)> {
         if let Some(a) = address_book.fulfilled_provisional_task_ids.pop_last() {
             trace!("expediate pop from provisional queue [rest: {}]", address_book.fulfilled_provisional_task_ids.len());
-            return Some((a.0, a.1));
+            panic!();
+            //return Some((a.0, a.1));
         }
 
         trace!("pop begin");

From 94e2b85ea3380375aee1a5064dc08f8d4b5b1c28 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 23 Aug 2022 15:26:56 +0900
Subject: [PATCH 1093/3199] save

---
 scheduler/src/lib.rs | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 8037659b7b848e..54e7c03cb71566 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -719,9 +719,10 @@ impl ScheduleStage {
                 return None;
                 continue;
             }
+            let lock_attempts = std::mem::take(next_task.tx.1);
 
             trace!("successful lock: (from_runnable: {}) after {} contentions", from_runnable, next_task.contention_count);
-            return Some((unique_weight, arc_next_task));
+            return Some((unique_weight, arc_next_task, lock_attempts));
         } else {
             break;
         }

From 7a5bf4b160f095798a9a807df419d467d4737b8a Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 23 Aug 2022 15:27:06 +0900
Subject: [PATCH 1094/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 54e7c03cb71566..c67510b9cc02cf 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -719,7 +719,7 @@ impl ScheduleStage {
                 return None;
                 continue;
             }
-            let lock_attempts = std::mem::take(next_task.tx.1);
+            let lock_attempts = std::mem::take(&mut next_task.tx.1);
 
             trace!("successful lock: (from_runnable: {}) after {} contentions", from_runnable, next_task.contention_count);
             return Some((unique_weight, arc_next_task, lock_attempts));

From 07e02311f2cc6499b0e2b2fa929a4e0cfdb69a3d Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 23 Aug 2022 15:27:38 +0900
Subject: [PATCH 1095/3199] save

---
 scheduler/src/lib.rs | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index c67510b9cc02cf..9cf0fa977409fb 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -812,6 +812,7 @@ impl ScheduleStage {
         unique_weight: UniqueWeight,
         task: TaskInQueue,
         contended_count: &mut usize,
+        lock_attempts: Vec<LockAttempt>,
     ) -> Box<ExecutionEnvironment> {
         let mut rng = rand::thread_rng();
         // load account now from AccountsDb
@@ -848,7 +849,7 @@ impl ScheduleStage {
     ) -> Option<Box<ExecutionEnvironment>> {
         let maybe_ee =
             Self::pop_from_queue_then_lock(runnable_queue, address_book, contended_count, prefer_immediate)
-                .map(|(uw, t)| Self::prepare_scheduled_execution(address_book, uw, t, contended_count));
+                .map(|(uw, t,ll)| Self::prepare_scheduled_execution(address_book, uw, t, contended_count, ll));
         maybe_ee
     }
 

From b93b9ed053c0bf1444a7878435243f9a9fcb0286 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 23 Aug 2022 15:27:46 +0900
Subject: [PATCH 1096/3199] save

---
 scheduler/src/lib.rs | 1 +
 1 file changed, 1 insertion(+)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 9cf0fa977409fb..9c4195ddc53001 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -823,6 +823,7 @@ impl ScheduleStage {
             task,
             unique_weight,
             cu: rng.gen_range(3, 1000),
+            lock_attempts,
         })
     }
 

From c4fc002aa952f62d952e5d2630066159a85c94cb Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 23 Aug 2022 15:28:54 +0900
Subject: [PATCH 1097/3199] save

---
 ledger-tool/src/main.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ledger-tool/src/main.rs b/ledger-tool/src/main.rs
index 5adfbb433e9d8d..41aad13f321643 100644
--- a/ledger-tool/src/main.rs
+++ b/ledger-tool/src/main.rs
@@ -348,7 +348,7 @@ fn output_slot(
                                     }
                                 }
                                 if found {
-                                    lock_attempt.heaviest_uncontended.store(Some(solana_scheduler::TaskInQueue::clone(task_cursor.value())));
+                                    lock_attempt.heaviest_uncontended = Some(solana_scheduler::TaskInQueue::clone(task_cursor.value()));
                                 }
                             }
                         }

From b42d2b4c251ecba51dff8f465f2f055fb947dc27 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 23 Aug 2022 15:29:22 +0900
Subject: [PATCH 1098/3199] save

---
 scheduler/src/lib.rs | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 9c4195ddc53001..71d1ce86094be2 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -72,7 +72,8 @@ pub struct LockAttempt {
     pub target: PageRc,
     status: LockStatus,
     requested_usage: RequestedUsage,
-    pub heaviest_uncontended: arc_swap::ArcSwapOption<Task>,
+    //pub heaviest_uncontended: arc_swap::ArcSwapOption<Task>,
+    pub heaviest_uncontended: Option<TaskInQueue>,
     //remembered: bool,
 }
 

From dae3ddb119d7674ef03e515b60a6772ca1fd19ec Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 23 Aug 2022 15:29:42 +0900
Subject: [PATCH 1099/3199] save

---
 scheduler/src/lib.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 71d1ce86094be2..7b95ad05e4cdb9 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -774,8 +774,8 @@ impl ScheduleStage {
 
             let page = l.target.page_mut();
             if newly_uncontended_while_queued && page.next_usage == Usage::Unused {
-                let maybe_task = l.heaviest_uncontended.load_full();
-                if let Some(task) = maybe_task {
+                //let maybe_task = l.heaviest_uncontended.load_full();
+                if let Some(task) = l.heaviest_uncontended { //maybe_task {
                     if task.currently_contended() {
                         address_book.uncontended_task_ids.insert(task.unique_weight, task);
                     }

From d766f75bb2991887a5714e24772c6b408268ba1f Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 23 Aug 2022 15:29:49 +0900
Subject: [PATCH 1100/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 7b95ad05e4cdb9..c8c7f1addca09d 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -775,7 +775,7 @@ impl ScheduleStage {
             let page = l.target.page_mut();
             if newly_uncontended_while_queued && page.next_usage == Usage::Unused {
                 //let maybe_task = l.heaviest_uncontended.load_full();
-                if let Some(task) = l.heaviest_uncontended { //maybe_task {
+                if let Some(task) = &l.heaviest_uncontended { //maybe_task {
                     if task.currently_contended() {
                         address_book.uncontended_task_ids.insert(task.unique_weight, task);
                     }

From 0023de9385a173bab7043af7a726b717c508c603 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 23 Aug 2022 15:30:26 +0900
Subject: [PATCH 1101/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index c8c7f1addca09d..39c4dc8bafc61d 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -775,7 +775,7 @@ impl ScheduleStage {
             let page = l.target.page_mut();
             if newly_uncontended_while_queued && page.next_usage == Usage::Unused {
                 //let maybe_task = l.heaviest_uncontended.load_full();
-                if let Some(task) = &l.heaviest_uncontended { //maybe_task {
+                if let Some(task) = l.heaviest_uncontended.take() { //maybe_task {
                     if task.currently_contended() {
                         address_book.uncontended_task_ids.insert(task.unique_weight, task);
                     }

From 4d089bb25083320414eedc4fdb8e3456bd1a0dd1 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 23 Aug 2022 15:31:34 +0900
Subject: [PATCH 1102/3199] save

---
 scheduler/src/lib.rs | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 39c4dc8bafc61d..9208965c5a8201 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -831,8 +831,7 @@ impl ScheduleStage {
     #[inline(never)]
     fn commit_result(ee: &mut ExecutionEnvironment, address_book: &mut AddressBook) {
         // do par()-ly?
-        let mut task = unsafe { TaskInQueue::get_mut_unchecked(&mut ee.task) };
-        Self::unlock_after_execution(address_book, &mut task.tx.1);
+        Self::unlock_after_execution(address_book, &mut ee.lock_attempts);
         // block-wide qos validation will be done here
         // if error risen..:
         //   don't commit the tx for banking and potentially finish scheduling at block max cu

From 8d58dd33edb4d34ba0d0d74b1ccad9633d617103 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 23 Aug 2022 15:33:01 +0900
Subject: [PATCH 1103/3199] save

---
 scheduler/src/lib.rs | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 9208965c5a8201..7936782e3b3c76 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -654,8 +654,7 @@ impl ScheduleStage {
     ) -> Option<(UniqueWeight, TaskInQueue, Vec<LockAttempt>)> {
         if let Some(a) = address_book.fulfilled_provisional_task_ids.pop_last() {
             trace!("expediate pop from provisional queue [rest: {}]", address_book.fulfilled_provisional_task_ids.len());
-            panic!();
-            //return Some((a.0, a.1));
+            return Some((a.0, a.1));
         }
 
         trace!("pop begin");

From a2d460e73cea534d3ad98c022dcdfcc8e9dc6515 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 23 Aug 2022 15:33:24 +0900
Subject: [PATCH 1104/3199] save

---
 scheduler/src/lib.rs | 1 +
 1 file changed, 1 insertion(+)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 7936782e3b3c76..09625d4446f893 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -654,6 +654,7 @@ impl ScheduleStage {
     ) -> Option<(UniqueWeight, TaskInQueue, Vec<LockAttempt>)> {
         if let Some(a) = address_book.fulfilled_provisional_task_ids.pop_last() {
             trace!("expediate pop from provisional queue [rest: {}]", address_book.fulfilled_provisional_task_ids.len());
+            let next_task = unsafe { TaskInQueue::get_mut_unchecked(&mut a.1) };
             return Some((a.0, a.1));
         }
 

From 8aff40f9e477524180a29a9bdd692c225cc8eb6d Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 23 Aug 2022 15:33:49 +0900
Subject: [PATCH 1105/3199] save

---
 scheduler/src/lib.rs | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 09625d4446f893..580d792a49dcd3 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -655,7 +655,8 @@ impl ScheduleStage {
         if let Some(a) = address_book.fulfilled_provisional_task_ids.pop_last() {
             trace!("expediate pop from provisional queue [rest: {}]", address_book.fulfilled_provisional_task_ids.len());
             let next_task = unsafe { TaskInQueue::get_mut_unchecked(&mut a.1) };
-            return Some((a.0, a.1));
+            let lock_attempts = std::mem::take(&mut next_task.tx.1);
+            return Some((a.0, a.1, lock_attempts));
         }
 
         trace!("pop begin");

From e6709220bffbe97444ce2d0df23fd0337b6e21a8 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 23 Aug 2022 15:34:09 +0900
Subject: [PATCH 1106/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 580d792a49dcd3..32686ef70fb5b8 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -652,7 +652,7 @@ impl ScheduleStage {
         contended_count: &mut usize,
         prefer_immediate: bool,
     ) -> Option<(UniqueWeight, TaskInQueue, Vec<LockAttempt>)> {
-        if let Some(a) = address_book.fulfilled_provisional_task_ids.pop_last() {
+        if let Some(mut a) = address_book.fulfilled_provisional_task_ids.pop_last() {
             trace!("expediate pop from provisional queue [rest: {}]", address_book.fulfilled_provisional_task_ids.len());
             let next_task = unsafe { TaskInQueue::get_mut_unchecked(&mut a.1) };
             let lock_attempts = std::mem::take(&mut next_task.tx.1);

From 8092eaa8f87cc6a29bc0e027dcb5bd9b8e3e9017 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 23 Aug 2022 15:35:36 +0900
Subject: [PATCH 1107/3199] save

---
 scheduler/Cargo.toml | 1 -
 1 file changed, 1 deletion(-)

diff --git a/scheduler/Cargo.toml b/scheduler/Cargo.toml
index daaff8ae54142e..893de70089827e 100644
--- a/scheduler/Cargo.toml
+++ b/scheduler/Cargo.toml
@@ -11,7 +11,6 @@ documentation = "https://docs.rs/solana-scheduler"
 publish = true
 
 [dependencies]
-arc-swap = "1.5.0"
 crossbeam-channel = "0.5.5"
 crossbeam-skiplist = { path = "../../../crossbeam/crossbeam-skiplist" }
 dashmap = { version = "4.0.2" }

From 5dc8f9e25eae37f41ec019bfdf50bbdcba7d5503 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 23 Aug 2022 15:37:22 +0900
Subject: [PATCH 1108/3199] save

---
 Cargo.lock | 1 -
 1 file changed, 1 deletion(-)

diff --git a/Cargo.lock b/Cargo.lock
index 393a336c9ffa0d..0c82122a5b7e51 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -6085,7 +6085,6 @@ dependencies = [
 name = "solana-scheduler"
 version = "1.11.5"
 dependencies = [
- "arc-swap",
  "crossbeam-channel",
  "crossbeam-skiplist",
  "dashmap",

From 3cd39484d6d979514324a99478e66b1d4444081d Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 23 Aug 2022 15:40:08 +0900
Subject: [PATCH 1109/3199] save

---
 ledger-tool/src/main.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ledger-tool/src/main.rs b/ledger-tool/src/main.rs
index 41aad13f321643..794003efe28bb8 100644
--- a/ledger-tool/src/main.rs
+++ b/ledger-tool/src/main.rs
@@ -428,7 +428,7 @@ fn output_slot(
                 let muxed_sender = muxed_sender.clone();
                 let t1 = std::thread::Builder::new()
                     .name(format!("sol-producer{}", thx))
-                    .spawn(move || loop {
+                    .spawn(move || {
                         for i in 0..loop_count {
                             error!("started!: {} {}", i, txes.len());
                             for tx in txes.iter().map(|t| Box::new((t.0.clone(), t.1.iter().map(|l| l.clone_for_test()).collect::<Vec<_>>()))) {

From 9443784375737df1683c1a5a340929bcd74dc4c8 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 23 Aug 2022 15:55:01 +0900
Subject: [PATCH 1110/3199] save

---
 transaction-scheduler-bench/src/main.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/transaction-scheduler-bench/src/main.rs b/transaction-scheduler-bench/src/main.rs
index 9505c2964236e6..39ca7d1811375f 100644
--- a/transaction-scheduler-bench/src/main.rs
+++ b/transaction-scheduler-bench/src/main.rs
@@ -343,7 +343,7 @@ fn handle_transaction_batch(
         .fetch_add(priority_collected, Ordering::Relaxed);
 
     let uq = transaction_batch.unique_weight;
-    for lock_attempt in transaction_batch.task.tx.1.iter() {
+    for mut lock_attempt in transaction_batch.lock_attempts.iter_mut() {
         let page = lock_attempt.target.page_ref();
         page.contended_unique_weights.remove_task_id(&uq);
         if let Some(mut task_cursor) = page.contended_unique_weights.heaviest_task_cursor() {
@@ -357,7 +357,7 @@ fn handle_transaction_batch(
                 }
             }
             if found {
-                lock_attempt.heaviest_uncontended.store(Some(solana_scheduler::TaskInQueue::clone(task_cursor.value())));
+                lock_attempt.heaviest_uncontended = Some(solana_scheduler::TaskInQueue::clone(task_cursor.value()));
             }
         }
     }

From 05fba84025910dbc44086ee4dca11a320c17a05d Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 23 Aug 2022 15:55:16 +0900
Subject: [PATCH 1111/3199] save

---
 transaction-scheduler-bench/src/main.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/transaction-scheduler-bench/src/main.rs b/transaction-scheduler-bench/src/main.rs
index 39ca7d1811375f..317055f33653a0 100644
--- a/transaction-scheduler-bench/src/main.rs
+++ b/transaction-scheduler-bench/src/main.rs
@@ -321,7 +321,7 @@ fn handle_transaction_batch(
     metrics: &TransactionSchedulerBenchMetrics,
     thread_index: usize,
     completed_transaction_sender: &(Sender<CompletedTransactionMessage>, Sender<solana_scheduler::Multiplexed>),
-    transaction_batch: TransactionBatchMessage,
+    mut transaction_batch: TransactionBatchMessage,
     execution_per_tx_us: u64,
 ) {
     let num_transactions = 1; //transaction_batch.len() as u64;

From e6fa17f59b719bffc9c74c6e6bb24098a7da8b63 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 23 Aug 2022 16:45:17 +0900
Subject: [PATCH 1112/3199] save

---
 scheduler/src/lib.rs | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 32686ef70fb5b8..f3471fe8133146 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -143,7 +143,8 @@ impl TaskIds {
 
     #[inline(never)]
     pub fn remove_task_id(&self, u: &TaskId) {
-        self.task_ids.remove(u);
+        let removed_now = self.task_ids.remove(u);
+        assert!(removed_now);
     }
 
     #[inline(never)]

From fcc9355db005381c7c3998ab4528d143e3b48ecf Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 23 Aug 2022 16:47:02 +0900
Subject: [PATCH 1113/3199] save

---
 scheduler/src/lib.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index f3471fe8133146..cc4ff3e5c28472 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -143,8 +143,8 @@ impl TaskIds {
 
     #[inline(never)]
     pub fn remove_task_id(&self, u: &TaskId) {
-        let removed_now = self.task_ids.remove(u);
-        assert!(removed_now);
+        let removed_entry = self.task_ids.remove(u);
+        assert!(removed_entry.is_removed());
     }
 
     #[inline(never)]

From abdc7597c6c6347e936ea39b839b7834f7e33c09 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 23 Aug 2022 16:47:16 +0900
Subject: [PATCH 1114/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index cc4ff3e5c28472..b80cdd560e8a15 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -144,7 +144,7 @@ impl TaskIds {
     #[inline(never)]
     pub fn remove_task_id(&self, u: &TaskId) {
         let removed_entry = self.task_ids.remove(u);
-        assert!(removed_entry.is_removed());
+        assert!(removed_entry.is_some());
     }
 
     #[inline(never)]

From bb2125078b7e011db69993010e661c18562d216a Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 23 Aug 2022 16:56:00 +0900
Subject: [PATCH 1115/3199] save

---
 scheduler/src/lib.rs | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index b80cdd560e8a15..52ec6fe73b1093 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -138,7 +138,12 @@ pub struct TaskIds {
 impl TaskIds {
     #[inline(never)]
     pub fn insert_task_id(&self, u: TaskId, task: TaskInQueue) {
-        self.task_ids.insert(u, task);
+        let is_inserted = false;
+        self.task_ids.get_or_insert_with(u, || {
+            is_inserted = true;
+            task
+        });
+        assert!(is_inserted);
     }
 
     #[inline(never)]

From 2d413b5be4545185293e1c948341e3ad013a5fbf Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 23 Aug 2022 16:56:09 +0900
Subject: [PATCH 1116/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 52ec6fe73b1093..3913b202f5c2b8 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -138,7 +138,7 @@ pub struct TaskIds {
 impl TaskIds {
     #[inline(never)]
     pub fn insert_task_id(&self, u: TaskId, task: TaskInQueue) {
-        let is_inserted = false;
+        let mut is_inserted = false;
         self.task_ids.get_or_insert_with(u, || {
             is_inserted = true;
             task

From 2e439cadd5670177d18a813eb3ab368d7a89d49d Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 23 Aug 2022 17:06:16 +0900
Subject: [PATCH 1117/3199] save

---
 scheduler/src/lib.rs | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 3913b202f5c2b8..74489a265352fc 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -434,7 +434,7 @@ pub struct Task {
     unique_weight: UniqueWeight,
     pub tx: Box<(SanitizedTransaction, Vec<LockAttempt>)>, // actually should be Bundle
     pub contention_count: usize,
-    pub uncontended: std::sync::atomic::AtomicUsize,
+    pub uncontended: usize,
 }
 
 impl Task {
@@ -452,15 +452,15 @@ impl Task {
     }
 
     pub fn currently_contended(&self) -> bool {
-        self.uncontended.load(std::sync::atomic::Ordering::SeqCst) == 1
+        self.uncontended == 1
     }
 
-    fn mark_as_contended(&self) {
-        self.uncontended.store(1, std::sync::atomic::Ordering::SeqCst)
+    fn mark_as_contended(&mut self) {
+        self.uncontended = 1;
     }
 
-    fn mark_as_uncontended(&self) {
-        self.uncontended.store(2, std::sync::atomic::Ordering::SeqCst)
+    fn mark_as_uncontended(&mut self) {
+        self.uncontended = 2;
     }
 }
 

From c0a4dd9ddc5f248641091deb016642851976f3ab Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 23 Aug 2022 17:07:21 +0900
Subject: [PATCH 1118/3199] save

---
 scheduler/src/lib.rs | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 74489a265352fc..22b26daaac1191 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -661,7 +661,10 @@ impl ScheduleStage {
         if let Some(mut a) = address_book.fulfilled_provisional_task_ids.pop_last() {
             trace!("expediate pop from provisional queue [rest: {}]", address_book.fulfilled_provisional_task_ids.len());
             let next_task = unsafe { TaskInQueue::get_mut_unchecked(&mut a.1) };
+
+            next_task.mark_as_uncontended();
             let lock_attempts = std::mem::take(&mut next_task.tx.1);
+
             return Some((a.0, a.1, lock_attempts));
         }
 
@@ -727,9 +730,12 @@ impl ScheduleStage {
                 return None;
                 continue;
             }
-            let lock_attempts = std::mem::take(&mut next_task.tx.1);
 
             trace!("successful lock: (from_runnable: {}) after {} contentions", from_runnable, next_task.contention_count);
+
+            next_task.mark_as_uncontended();
+            let lock_attempts = std::mem::take(&mut next_task.tx.1);
+
             return Some((unique_weight, arc_next_task, lock_attempts));
         } else {
             break;
@@ -824,7 +830,6 @@ impl ScheduleStage {
     ) -> Box<ExecutionEnvironment> {
         let mut rng = rand::thread_rng();
         // load account now from AccountsDb
-        task.mark_as_uncontended();
         *contended_count -= 1;
 
         Box::new(ExecutionEnvironment {

From 5f6f21b10873ad688d5dfc2bb50e167b0a660719 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 23 Aug 2022 17:19:08 +0900
Subject: [PATCH 1119/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 22b26daaac1191..7245489a29b03d 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -702,7 +702,7 @@ impl ScheduleStage {
                 if from_runnable {
                     trace!("move to contended due to lock failure [{}/{}/{}]", unlockable_count, provisional_count, lock_count);
                     next_task.mark_as_contended();
-                    *contended_count += 1;
+                    *contended_count = contention_count.checked_add(1).unwrap();
                     // maybe run lightweight prune logic on contended_queue here.
                 } else {
                     trace!("relock failed [{}/{}/{}]; remains in contended: {:?} contention: {}", unlockable_count, provisional_count, lock_count, &unique_weight, next_task.contention_count);

From 2f91b9988fbb76db3cde66eeb46dd43385312960 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 23 Aug 2022 17:19:25 +0900
Subject: [PATCH 1120/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 7245489a29b03d..524b6e22a4395d 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -702,7 +702,7 @@ impl ScheduleStage {
                 if from_runnable {
                     trace!("move to contended due to lock failure [{}/{}/{}]", unlockable_count, provisional_count, lock_count);
                     next_task.mark_as_contended();
-                    *contended_count = contention_count.checked_add(1).unwrap();
+                    *contended_count = contended_count.checked_add(1).unwrap();
                     // maybe run lightweight prune logic on contended_queue here.
                 } else {
                     trace!("relock failed [{}/{}/{}]; remains in contended: {:?} contention: {}", unlockable_count, provisional_count, lock_count, &unique_weight, next_task.contention_count);

From 434aa45b37351caa9b6bf1b77e0a0eb9fe6ffff9 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 23 Aug 2022 17:20:38 +0900
Subject: [PATCH 1121/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 524b6e22a4395d..3754ad979ef140 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -830,7 +830,7 @@ impl ScheduleStage {
     ) -> Box<ExecutionEnvironment> {
         let mut rng = rand::thread_rng();
         // load account now from AccountsDb
-        *contended_count -= 1;
+        *contended_count = contended_count.checked_sub(1).unwrap();
 
         Box::new(ExecutionEnvironment {
             task,

From 8c9441f2404f2ea5e69daa47ed53eb03b67be627 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 23 Aug 2022 17:21:43 +0900
Subject: [PATCH 1122/3199] save

---
 scheduler/src/lib.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 3754ad979ef140..68e7560d40180e 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -662,6 +662,7 @@ impl ScheduleStage {
             trace!("expediate pop from provisional queue [rest: {}]", address_book.fulfilled_provisional_task_ids.len());
             let next_task = unsafe { TaskInQueue::get_mut_unchecked(&mut a.1) };
 
+            *contended_count = contended_count.checked_sub(1).unwrap();
             next_task.mark_as_uncontended();
             let lock_attempts = std::mem::take(&mut next_task.tx.1);
 
@@ -733,6 +734,7 @@ impl ScheduleStage {
 
             trace!("successful lock: (from_runnable: {}) after {} contentions", from_runnable, next_task.contention_count);
 
+            *contended_count = contended_count.checked_sub(1).unwrap();
             next_task.mark_as_uncontended();
             let lock_attempts = std::mem::take(&mut next_task.tx.1);
 
@@ -825,12 +827,10 @@ impl ScheduleStage {
         address_book: &mut AddressBook,
         unique_weight: UniqueWeight,
         task: TaskInQueue,
-        contended_count: &mut usize,
         lock_attempts: Vec<LockAttempt>,
     ) -> Box<ExecutionEnvironment> {
         let mut rng = rand::thread_rng();
         // load account now from AccountsDb
-        *contended_count = contended_count.checked_sub(1).unwrap();
 
         Box::new(ExecutionEnvironment {
             task,

From e65ed0fa532eea558a737d10f02764f3b086265a Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 23 Aug 2022 17:22:00 +0900
Subject: [PATCH 1123/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 68e7560d40180e..5004746429ed74 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -862,7 +862,7 @@ impl ScheduleStage {
     ) -> Option<Box<ExecutionEnvironment>> {
         let maybe_ee =
             Self::pop_from_queue_then_lock(runnable_queue, address_book, contended_count, prefer_immediate)
-                .map(|(uw, t,ll)| Self::prepare_scheduled_execution(address_book, uw, t, contended_count, ll));
+                .map(|(uw, t,ll)| Self::prepare_scheduled_execution(address_book, uw, t, ll));
         maybe_ee
     }
 

From 892200f565fb26f8af6e682db3581694184db238 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 23 Aug 2022 17:23:21 +0900
Subject: [PATCH 1124/3199] save

---
 scheduler/src/lib.rs | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 5004746429ed74..ccfbac735e9a10 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -734,7 +734,9 @@ impl ScheduleStage {
 
             trace!("successful lock: (from_runnable: {}) after {} contentions", from_runnable, next_task.contention_count);
 
-            *contended_count = contended_count.checked_sub(1).unwrap();
+            if !from_runnable {
+                *contended_count = contended_count.checked_sub(1).unwrap();
+            }
             next_task.mark_as_uncontended();
             let lock_attempts = std::mem::take(&mut next_task.tx.1);
 

From e295d261da408102a9579e170e75193b85bd75ac Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 23 Aug 2022 21:23:14 +0900
Subject: [PATCH 1125/3199] Revert "save"

This reverts commit 2e439cadd5670177d18a813eb3ab368d7a89d49d.
---
 scheduler/src/lib.rs | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index ccfbac735e9a10..991d90485443c8 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -434,7 +434,7 @@ pub struct Task {
     unique_weight: UniqueWeight,
     pub tx: Box<(SanitizedTransaction, Vec<LockAttempt>)>, // actually should be Bundle
     pub contention_count: usize,
-    pub uncontended: usize,
+    pub uncontended: std::sync::atomic::AtomicUsize,
 }
 
 impl Task {
@@ -452,15 +452,15 @@ impl Task {
     }
 
     pub fn currently_contended(&self) -> bool {
-        self.uncontended == 1
+        self.uncontended.load(std::sync::atomic::Ordering::SeqCst) == 1
     }
 
-    fn mark_as_contended(&mut self) {
-        self.uncontended = 1;
+    fn mark_as_contended(&self) {
+        self.uncontended.store(1, std::sync::atomic::Ordering::SeqCst)
     }
 
-    fn mark_as_uncontended(&mut self) {
-        self.uncontended = 2;
+    fn mark_as_uncontended(&self) {
+        self.uncontended.store(2, std::sync::atomic::Ordering::SeqCst)
     }
 }
 

From 72a492ddf865fa453cc3e04957e5845014ce2746 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 23 Aug 2022 21:24:16 +0900
Subject: [PATCH 1126/3199] save

---
 transaction-scheduler-bench/src/main.rs | 1 +
 1 file changed, 1 insertion(+)

diff --git a/transaction-scheduler-bench/src/main.rs b/transaction-scheduler-bench/src/main.rs
index 317055f33653a0..444d7c9a82dae8 100644
--- a/transaction-scheduler-bench/src/main.rs
+++ b/transaction-scheduler-bench/src/main.rs
@@ -350,6 +350,7 @@ fn handle_transaction_batch(
             let mut found = true;
             while !task_cursor.value().currently_contended() {
                 if let Some(new_cursor) = task_cursor.prev() {
+                    assert!(new_cursor.key() < task_cursor.key());
                     task_cursor = new_cursor;
                 } else {
                     found = false;

From ea9d92660335ccb829ebc15910697f10a5352ea5 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 23 Aug 2022 21:26:17 +0900
Subject: [PATCH 1127/3199] save

---
 scheduler/src/lib.rs | 1 +
 1 file changed, 1 insertion(+)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 991d90485443c8..6713a1a68e39f0 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -896,6 +896,7 @@ impl ScheduleStage {
         } else {
             let h = std::thread::Builder::new().name("sol-reaper".to_string()).spawn(move || {
                 while let a = ee_receiver.recv().unwrap() {
+                    a.tx.1.len() == 0;
                     drop(a);
                 }
             }).unwrap();

From f31e8963619b9424014907012be7e023521ac7b5 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 23 Aug 2022 21:26:30 +0900
Subject: [PATCH 1128/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 6713a1a68e39f0..6731f0917ad0b9 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -896,7 +896,7 @@ impl ScheduleStage {
         } else {
             let h = std::thread::Builder::new().name("sol-reaper".to_string()).spawn(move || {
                 while let a = ee_receiver.recv().unwrap() {
-                    a.tx.1.len() == 0;
+                    a.task.tx.1.is_empty() == 0;
                     drop(a);
                 }
             }).unwrap();

From 4c209d971bb7fa571856e9901efff439367f7026 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 23 Aug 2022 21:26:45 +0900
Subject: [PATCH 1129/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 6731f0917ad0b9..1994040794ce6f 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -896,7 +896,7 @@ impl ScheduleStage {
         } else {
             let h = std::thread::Builder::new().name("sol-reaper".to_string()).spawn(move || {
                 while let a = ee_receiver.recv().unwrap() {
-                    a.task.tx.1.is_empty() == 0;
+                    assert!(a.task.tx.1.is_empty());
                     drop(a);
                 }
             }).unwrap();

From f01b91dc369d8444c8f53acf9783e6f5468e0793 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 23 Aug 2022 21:27:31 +0900
Subject: [PATCH 1130/3199] save

---
 scheduler/src/lib.rs | 1 +
 1 file changed, 1 insertion(+)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 1994040794ce6f..93f2c16a79cd79 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -897,6 +897,7 @@ impl ScheduleStage {
             let h = std::thread::Builder::new().name("sol-reaper".to_string()).spawn(move || {
                 while let a = ee_receiver.recv().unwrap() {
                     assert!(a.task.tx.1.is_empty());
+                    drop(a.lock_attempts);
                     drop(a);
                 }
             }).unwrap();

From a006e055c12a2b206077f7bcc301e5b3c1d801fe Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 23 Aug 2022 21:28:01 +0900
Subject: [PATCH 1131/3199] save

---
 scheduler/src/lib.rs | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 93f2c16a79cd79..6d406b15a84398 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -897,8 +897,7 @@ impl ScheduleStage {
             let h = std::thread::Builder::new().name("sol-reaper".to_string()).spawn(move || {
                 while let a = ee_receiver.recv().unwrap() {
                     assert!(a.task.tx.1.is_empty());
-                    drop(a.lock_attempts);
-                    drop(a);
+                    let {lock_attempts, ..} = a;
                 }
             }).unwrap();
 

From 18f9b60287d474af7067c51872d202134c72fe02 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 23 Aug 2022 21:28:11 +0900
Subject: [PATCH 1132/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 6d406b15a84398..41ee3197e15a6f 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -897,7 +897,7 @@ impl ScheduleStage {
             let h = std::thread::Builder::new().name("sol-reaper".to_string()).spawn(move || {
                 while let a = ee_receiver.recv().unwrap() {
                     assert!(a.task.tx.1.is_empty());
-                    let {lock_attempts, ..} = a;
+                    let ExecutionEnvironment {lock_attempts, ..} = a;
                 }
             }).unwrap();
 

From 7b50d6de2c4f62ff9a010258fdacdd5205835a2b Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 23 Aug 2022 21:28:22 +0900
Subject: [PATCH 1133/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 41ee3197e15a6f..f522044a0d1cb1 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -897,7 +897,7 @@ impl ScheduleStage {
             let h = std::thread::Builder::new().name("sol-reaper".to_string()).spawn(move || {
                 while let a = ee_receiver.recv().unwrap() {
                     assert!(a.task.tx.1.is_empty());
-                    let ExecutionEnvironment {lock_attempts, ..} = a;
+                    let Box<ExecutionEnvironment> {lock_attempts, ..} = a;
                 }
             }).unwrap();
 

From fce29ad7fac01a4330bb39eac35b75e354d48187 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 23 Aug 2022 21:28:42 +0900
Subject: [PATCH 1134/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index f522044a0d1cb1..9ee6590cfb2224 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -897,7 +897,7 @@ impl ScheduleStage {
             let h = std::thread::Builder::new().name("sol-reaper".to_string()).spawn(move || {
                 while let a = ee_receiver.recv().unwrap() {
                     assert!(a.task.tx.1.is_empty());
-                    let Box<ExecutionEnvironment> {lock_attempts, ..} = a;
+                    let ExecutionEnvironment {lock_attempts, ..} = a.into();
                 }
             }).unwrap();
 

From 9cd3cb564e0c2fb5595568646b6df46ba6b98bce Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 23 Aug 2022 21:29:19 +0900
Subject: [PATCH 1135/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 9ee6590cfb2224..64fbe97b3cd96b 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -897,7 +897,7 @@ impl ScheduleStage {
             let h = std::thread::Builder::new().name("sol-reaper".to_string()).spawn(move || {
                 while let a = ee_receiver.recv().unwrap() {
                     assert!(a.task.tx.1.is_empty());
-                    let ExecutionEnvironment {lock_attempts, ..} = a.into();
+                    let lock_attempts = std::mem::take(&mut a.lock_attempts);
                 }
             }).unwrap();
 

From 87c6e658dda0a1581a9317ade4b008aa74e75b9b Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 23 Aug 2022 21:29:25 +0900
Subject: [PATCH 1136/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 64fbe97b3cd96b..30bd46a7ec735e 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -895,7 +895,7 @@ impl ScheduleStage {
             (to_next_stage, None)
         } else {
             let h = std::thread::Builder::new().name("sol-reaper".to_string()).spawn(move || {
-                while let a = ee_receiver.recv().unwrap() {
+                while let mut a = ee_receiver.recv().unwrap() {
                     assert!(a.task.tx.1.is_empty());
                     let lock_attempts = std::mem::take(&mut a.lock_attempts);
                 }

From 03555e29dfab5db77f8f1a681747f6ed5ea8cf25 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 23 Aug 2022 21:30:38 +0900
Subject: [PATCH 1137/3199] save

---
 scheduler/src/lib.rs | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 30bd46a7ec735e..50453a52e9630a 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -897,7 +897,9 @@ impl ScheduleStage {
             let h = std::thread::Builder::new().name("sol-reaper".to_string()).spawn(move || {
                 while let mut a = ee_receiver.recv().unwrap() {
                     assert!(a.task.tx.1.is_empty());
-                    let lock_attempts = std::mem::take(&mut a.lock_attempts);
+                    //let lock_attempts = std::mem::take(&mut a.lock_attempts);
+                    //drop(lock_attempts);
+                    TaskInQueue::into(a.task)
                 }
             }).unwrap();
 

From dcb40d7b7e31ae90a8d8d7ab55b948ae22b3d122 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 23 Aug 2022 21:30:56 +0900
Subject: [PATCH 1138/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 50453a52e9630a..0cb85aa6d81a39 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -899,7 +899,7 @@ impl ScheduleStage {
                     assert!(a.task.tx.1.is_empty());
                     //let lock_attempts = std::mem::take(&mut a.lock_attempts);
                     //drop(lock_attempts);
-                    TaskInQueue::into(a.task)
+                    TaskInQueue::get_mut(a.task)
                 }
             }).unwrap();
 

From f287bcc6c60708cbac40ffbb3550ec366f44faf8 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 23 Aug 2022 21:31:05 +0900
Subject: [PATCH 1139/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 0cb85aa6d81a39..c97f584f49e6b2 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -899,7 +899,7 @@ impl ScheduleStage {
                     assert!(a.task.tx.1.is_empty());
                     //let lock_attempts = std::mem::take(&mut a.lock_attempts);
                     //drop(lock_attempts);
-                    TaskInQueue::get_mut(a.task)
+                    TaskInQueue::get_mut(&mut a.task)
                 }
             }).unwrap();
 

From 71a7b6524c8dc9aa868779cf5c5739796af542ce Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 23 Aug 2022 21:34:43 +0900
Subject: [PATCH 1140/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index c97f584f49e6b2..5c2eb0ae7fd0dd 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -899,7 +899,7 @@ impl ScheduleStage {
                     assert!(a.task.tx.1.is_empty());
                     //let lock_attempts = std::mem::take(&mut a.lock_attempts);
                     //drop(lock_attempts);
-                    TaskInQueue::get_mut(&mut a.task)
+                    TaskInQueue::get_mut(&mut a.task).unwrap()
                 }
             }).unwrap();
 

From 68f03f777884be20475df1ea42e191511fcc4cd0 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 23 Aug 2022 21:34:51 +0900
Subject: [PATCH 1141/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 5c2eb0ae7fd0dd..71b70b8b4a353b 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -899,7 +899,7 @@ impl ScheduleStage {
                     assert!(a.task.tx.1.is_empty());
                     //let lock_attempts = std::mem::take(&mut a.lock_attempts);
                     //drop(lock_attempts);
-                    TaskInQueue::get_mut(&mut a.task).unwrap()
+                    TaskInQueue::get_mut(&mut a.task).unwrap();
                 }
             }).unwrap();
 

From 0ec3a15f60014ba6ef857827c880dfd523c90b84 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 23 Aug 2022 21:37:03 +0900
Subject: [PATCH 1142/3199] save

---
 scheduler/src/lib.rs                    | 2 +-
 transaction-scheduler-bench/src/main.rs | 2 ++
 2 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 71b70b8b4a353b..906c421737c5ae 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -899,7 +899,7 @@ impl ScheduleStage {
                     assert!(a.task.tx.1.is_empty());
                     //let lock_attempts = std::mem::take(&mut a.lock_attempts);
                     //drop(lock_attempts);
-                    TaskInQueue::get_mut(&mut a.task).unwrap();
+                    //TaskInQueue::get_mut(&mut a.task).unwrap();
                 }
             }).unwrap();
 
diff --git a/transaction-scheduler-bench/src/main.rs b/transaction-scheduler-bench/src/main.rs
index 444d7c9a82dae8..f99e18de642480 100644
--- a/transaction-scheduler-bench/src/main.rs
+++ b/transaction-scheduler-bench/src/main.rs
@@ -348,9 +348,11 @@ fn handle_transaction_batch(
         page.contended_unique_weights.remove_task_id(&uq);
         if let Some(mut task_cursor) = page.contended_unique_weights.heaviest_task_cursor() {
             let mut found = true;
+            assert_ne!(task_cursor.key(), uq);
             while !task_cursor.value().currently_contended() {
                 if let Some(new_cursor) = task_cursor.prev() {
                     assert!(new_cursor.key() < task_cursor.key());
+                    assert_ne!(new_cursor.key(), uq);
                     task_cursor = new_cursor;
                 } else {
                     found = false;

From 4dfde7940df45af0177c6d0eb67f651de17c0ef7 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 23 Aug 2022 21:37:34 +0900
Subject: [PATCH 1143/3199] save

---
 transaction-scheduler-bench/src/main.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/transaction-scheduler-bench/src/main.rs b/transaction-scheduler-bench/src/main.rs
index f99e18de642480..dad7f1915c4642 100644
--- a/transaction-scheduler-bench/src/main.rs
+++ b/transaction-scheduler-bench/src/main.rs
@@ -352,7 +352,7 @@ fn handle_transaction_batch(
             while !task_cursor.value().currently_contended() {
                 if let Some(new_cursor) = task_cursor.prev() {
                     assert!(new_cursor.key() < task_cursor.key());
-                    assert_ne!(new_cursor.key(), uq);
+                    assert_ne!(new_cursor.key(), &uq);
                     task_cursor = new_cursor;
                 } else {
                     found = false;

From 7b86ab2b0d6828ae2a2d449ae43920982f7b8a13 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 23 Aug 2022 21:37:44 +0900
Subject: [PATCH 1144/3199] save

---
 transaction-scheduler-bench/src/main.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/transaction-scheduler-bench/src/main.rs b/transaction-scheduler-bench/src/main.rs
index dad7f1915c4642..02e6cba00ef77b 100644
--- a/transaction-scheduler-bench/src/main.rs
+++ b/transaction-scheduler-bench/src/main.rs
@@ -352,7 +352,7 @@ fn handle_transaction_batch(
             while !task_cursor.value().currently_contended() {
                 if let Some(new_cursor) = task_cursor.prev() {
                     assert!(new_cursor.key() < task_cursor.key());
-                    assert_ne!(new_cursor.key(), &uq);
+                    assert_ne!(*new_cursor.key(), uq);
                     task_cursor = new_cursor;
                 } else {
                     found = false;

From 3058dc3ed2f1d08573192266dd91c1f8378c7ebc Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 23 Aug 2022 21:37:53 +0900
Subject: [PATCH 1145/3199] save

---
 transaction-scheduler-bench/src/main.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/transaction-scheduler-bench/src/main.rs b/transaction-scheduler-bench/src/main.rs
index 02e6cba00ef77b..5c77bd3fd46a54 100644
--- a/transaction-scheduler-bench/src/main.rs
+++ b/transaction-scheduler-bench/src/main.rs
@@ -352,7 +352,7 @@ fn handle_transaction_batch(
             while !task_cursor.value().currently_contended() {
                 if let Some(new_cursor) = task_cursor.prev() {
                     assert!(new_cursor.key() < task_cursor.key());
-                    assert_ne!(*new_cursor.key(), uq);
+                    assert_ne!(*new_cursor.key(), *uq);
                     task_cursor = new_cursor;
                 } else {
                     found = false;

From 9b2ba2c7ebd1f5646e4431a42d174d7b2db4377b Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 23 Aug 2022 21:38:19 +0900
Subject: [PATCH 1146/3199] save

---
 transaction-scheduler-bench/src/main.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/transaction-scheduler-bench/src/main.rs b/transaction-scheduler-bench/src/main.rs
index 5c77bd3fd46a54..c72e9f78e0a1e6 100644
--- a/transaction-scheduler-bench/src/main.rs
+++ b/transaction-scheduler-bench/src/main.rs
@@ -348,11 +348,11 @@ fn handle_transaction_batch(
         page.contended_unique_weights.remove_task_id(&uq);
         if let Some(mut task_cursor) = page.contended_unique_weights.heaviest_task_cursor() {
             let mut found = true;
-            assert_ne!(task_cursor.key(), uq);
+            assert_ne!(task_cursor.key(), &uq);
             while !task_cursor.value().currently_contended() {
                 if let Some(new_cursor) = task_cursor.prev() {
                     assert!(new_cursor.key() < task_cursor.key());
-                    assert_ne!(*new_cursor.key(), *uq);
+                    assert_ne!(new_cursor.key(), &uq);
                     task_cursor = new_cursor;
                 } else {
                     found = false;

From b8f5d282ea0c8c34459275fbcd3913dd755b34b6 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 23 Aug 2022 21:47:41 +0900
Subject: [PATCH 1147/3199] save

---
 scheduler/src/lib.rs | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 906c421737c5ae..7421bb762ac933 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -462,6 +462,10 @@ impl Task {
     fn mark_as_uncontended(&self) {
         self.uncontended.store(2, std::sync::atomic::Ordering::SeqCst)
     }
+
+    fn mark_as_finished(&self) {
+        self.uncontended.store(3, std::sync::atomic::Ordering::SeqCst)
+    }
 }
 
 // RunnableQueue, ContendedQueue?
@@ -846,6 +850,7 @@ impl ScheduleStage {
     fn commit_result(ee: &mut ExecutionEnvironment, address_book: &mut AddressBook) {
         // do par()-ly?
         Self::unlock_after_execution(address_book, &mut ee.lock_attempts);
+        ee.task.mark_as_finished();
         // block-wide qos validation will be done here
         // if error risen..:
         //   don't commit the tx for banking and potentially finish scheduling at block max cu

From 129d133af05bb0b1c3be6fc89443a5db33873714 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 23 Aug 2022 21:48:48 +0900
Subject: [PATCH 1148/3199] save

---
 scheduler/src/lib.rs | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 7421bb762ac933..ea989eb6280880 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -455,6 +455,10 @@ impl Task {
         self.uncontended.load(std::sync::atomic::Ordering::SeqCst) == 1
     }
 
+    pub fn already_finished(&self) -> bool {
+        self.uncontended.load(std::sync::atomic::Ordering::SeqCst) == 3
+    }
+
     fn mark_as_contended(&self) {
         self.uncontended.store(1, std::sync::atomic::Ordering::SeqCst)
     }
@@ -797,6 +801,7 @@ impl ScheduleStage {
             if newly_uncontended_while_queued && page.next_usage == Usage::Unused {
                 //let maybe_task = l.heaviest_uncontended.load_full();
                 if let Some(task) = l.heaviest_uncontended.take() { //maybe_task {
+                    assert!(!task.already_finished());
                     if task.currently_contended() {
                         address_book.uncontended_task_ids.insert(task.unique_weight, task);
                     }

From c03eb6ded247534724c565e7ef64ac4657a6ddef Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 23 Aug 2022 21:54:35 +0900
Subject: [PATCH 1149/3199] save

---
 scheduler/src/lib.rs | 15 +++------------
 1 file changed, 3 insertions(+), 12 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index ea989eb6280880..fc933ff8091a23 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -152,11 +152,6 @@ impl TaskIds {
         assert!(removed_entry.is_some());
     }
 
-    #[inline(never)]
-    fn has_more(&self) -> bool {
-        !self.task_ids.is_empty()
-    }
-
     #[inline(never)]
     fn heaviest_task_id(&self) -> Option<TaskId> {
         self.task_ids.back().map(|e| *(e.key()))
@@ -328,7 +323,6 @@ impl AddressBook {
         //debug_assert!(attempt.is_success());
 
         let mut newly_uncontended = false;
-        let mut still_queued = false;
 
         let mut page = attempt.target.page_mut();
 
@@ -354,12 +348,9 @@ impl AddressBook {
 
         if newly_uncontended {
             page.current_usage = Usage::Unused;
-            if page.contended_unique_weights.has_more() {
-                still_queued = true;
-            }
         }
 
-        still_queued
+        newly_uncontended
     }
 
     #[inline(never)]
@@ -795,10 +786,10 @@ impl ScheduleStage {
     #[inline(never)]
     fn unlock_after_execution(address_book: &mut AddressBook, lock_attempts: &mut Vec<LockAttempt>) {
         for mut l in lock_attempts.into_iter() {
-            let newly_uncontended_while_queued = address_book.reset_lock(&mut l, true);
+            let newly_uncontended = address_book.reset_lock(&mut l, true);
 
             let page = l.target.page_mut();
-            if newly_uncontended_while_queued && page.next_usage == Usage::Unused {
+            if newly_uncontended && page.next_usage == Usage::Unused {
                 //let maybe_task = l.heaviest_uncontended.load_full();
                 if let Some(task) = l.heaviest_uncontended.take() { //maybe_task {
                     assert!(!task.already_finished());

From a3a88ce0f66e93df326f0034bce7fce2ea56697f Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 23 Aug 2022 21:55:40 +0900
Subject: [PATCH 1150/3199] save

---
 scheduler/src/lib.rs | 15 ++++++++++++---
 1 file changed, 12 insertions(+), 3 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index fc933ff8091a23..ea989eb6280880 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -152,6 +152,11 @@ impl TaskIds {
         assert!(removed_entry.is_some());
     }
 
+    #[inline(never)]
+    fn has_more(&self) -> bool {
+        !self.task_ids.is_empty()
+    }
+
     #[inline(never)]
     fn heaviest_task_id(&self) -> Option<TaskId> {
         self.task_ids.back().map(|e| *(e.key()))
@@ -323,6 +328,7 @@ impl AddressBook {
         //debug_assert!(attempt.is_success());
 
         let mut newly_uncontended = false;
+        let mut still_queued = false;
 
         let mut page = attempt.target.page_mut();
 
@@ -348,9 +354,12 @@ impl AddressBook {
 
         if newly_uncontended {
             page.current_usage = Usage::Unused;
+            if page.contended_unique_weights.has_more() {
+                still_queued = true;
+            }
         }
 
-        newly_uncontended
+        still_queued
     }
 
     #[inline(never)]
@@ -786,10 +795,10 @@ impl ScheduleStage {
     #[inline(never)]
     fn unlock_after_execution(address_book: &mut AddressBook, lock_attempts: &mut Vec<LockAttempt>) {
         for mut l in lock_attempts.into_iter() {
-            let newly_uncontended = address_book.reset_lock(&mut l, true);
+            let newly_uncontended_while_queued = address_book.reset_lock(&mut l, true);
 
             let page = l.target.page_mut();
-            if newly_uncontended && page.next_usage == Usage::Unused {
+            if newly_uncontended_while_queued && page.next_usage == Usage::Unused {
                 //let maybe_task = l.heaviest_uncontended.load_full();
                 if let Some(task) = l.heaviest_uncontended.take() { //maybe_task {
                     assert!(!task.already_finished());

From 60d7178302468235c6391031d77cd96425aff16b Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 23 Aug 2022 21:57:12 +0900
Subject: [PATCH 1151/3199] save

---
 scheduler/src/lib.rs | 15 +++------------
 1 file changed, 3 insertions(+), 12 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index ea989eb6280880..fc933ff8091a23 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -152,11 +152,6 @@ impl TaskIds {
         assert!(removed_entry.is_some());
     }
 
-    #[inline(never)]
-    fn has_more(&self) -> bool {
-        !self.task_ids.is_empty()
-    }
-
     #[inline(never)]
     fn heaviest_task_id(&self) -> Option<TaskId> {
         self.task_ids.back().map(|e| *(e.key()))
@@ -328,7 +323,6 @@ impl AddressBook {
         //debug_assert!(attempt.is_success());
 
         let mut newly_uncontended = false;
-        let mut still_queued = false;
 
         let mut page = attempt.target.page_mut();
 
@@ -354,12 +348,9 @@ impl AddressBook {
 
         if newly_uncontended {
             page.current_usage = Usage::Unused;
-            if page.contended_unique_weights.has_more() {
-                still_queued = true;
-            }
         }
 
-        still_queued
+        newly_uncontended
     }
 
     #[inline(never)]
@@ -795,10 +786,10 @@ impl ScheduleStage {
     #[inline(never)]
     fn unlock_after_execution(address_book: &mut AddressBook, lock_attempts: &mut Vec<LockAttempt>) {
         for mut l in lock_attempts.into_iter() {
-            let newly_uncontended_while_queued = address_book.reset_lock(&mut l, true);
+            let newly_uncontended = address_book.reset_lock(&mut l, true);
 
             let page = l.target.page_mut();
-            if newly_uncontended_while_queued && page.next_usage == Usage::Unused {
+            if newly_uncontended && page.next_usage == Usage::Unused {
                 //let maybe_task = l.heaviest_uncontended.load_full();
                 if let Some(task) = l.heaviest_uncontended.take() { //maybe_task {
                     assert!(!task.already_finished());

From 1935490cf2ba9bff7b186b2f8f5e0e8cbc877511 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 23 Aug 2022 21:58:43 +0900
Subject: [PATCH 1152/3199] save

---
 scheduler/src/lib.rs | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index fc933ff8091a23..744e75079ab0c3 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -790,8 +790,7 @@ impl ScheduleStage {
 
             let page = l.target.page_mut();
             if newly_uncontended && page.next_usage == Usage::Unused {
-                //let maybe_task = l.heaviest_uncontended.load_full();
-                if let Some(task) = l.heaviest_uncontended.take() { //maybe_task {
+                if let Some(task) = l.heaviest_uncontended.take() {
                     assert!(!task.already_finished());
                     if task.currently_contended() {
                         address_book.uncontended_task_ids.insert(task.unique_weight, task);

From 2f20385ab7024c27b5aa73ec6b682c912f816951 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 23 Aug 2022 22:00:11 +0900
Subject: [PATCH 1153/3199] save

---
 scheduler/src/lib.rs | 1 +
 1 file changed, 1 insertion(+)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 744e75079ab0c3..bd8d812441ed46 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -459,6 +459,7 @@ impl Task {
     }
 
     fn mark_as_finished(&self) {
+        assert!(self.already_finished());
         self.uncontended.store(3, std::sync::atomic::Ordering::SeqCst)
     }
 }

From beace60f19f66d0b8e3e4a4194996bb714920bf2 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 23 Aug 2022 22:01:18 +0900
Subject: [PATCH 1154/3199] save

---
 scheduler/src/lib.rs | 1 +
 1 file changed, 1 insertion(+)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index bd8d812441ed46..b260f74a86646b 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -455,6 +455,7 @@ impl Task {
     }
 
     fn mark_as_uncontended(&self) {
+        assert!(self.currently_contended());
         self.uncontended.store(2, std::sync::atomic::Ordering::SeqCst)
     }
 

From 50e9f560cedd6b827752ce29a460112545ce955c Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 23 Aug 2022 22:03:48 +0900
Subject: [PATCH 1155/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index b260f74a86646b..ba33bab84ceb3f 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -737,8 +737,8 @@ impl ScheduleStage {
 
             if !from_runnable {
                 *contended_count = contended_count.checked_sub(1).unwrap();
+                next_task.mark_as_uncontended();
             }
-            next_task.mark_as_uncontended();
             let lock_attempts = std::mem::take(&mut next_task.tx.1);
 
             return Some((unique_weight, arc_next_task, lock_attempts));

From 714f35845dd9b4da5ec5e41f540267ac41c38364 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 23 Aug 2022 22:04:52 +0900
Subject: [PATCH 1156/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index ba33bab84ceb3f..baa691e659fe7b 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -460,7 +460,7 @@ impl Task {
     }
 
     fn mark_as_finished(&self) {
-        assert!(self.already_finished());
+        assert!(!self.already_finished());
         self.uncontended.store(3, std::sync::atomic::Ordering::SeqCst)
     }
 }

From 314f2bb08d2df6ea50aa4e7afdfce777315de61c Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 23 Aug 2022 22:09:00 +0900
Subject: [PATCH 1157/3199] save

---
 scheduler/src/lib.rs | 1 +
 1 file changed, 1 insertion(+)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index baa691e659fe7b..f2ba7486f62154 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -461,6 +461,7 @@ impl Task {
 
     fn mark_as_finished(&self) {
         assert!(!self.already_finished());
+        assert!(!self.currently_contended());
         self.uncontended.store(3, std::sync::atomic::Ordering::SeqCst)
     }
 }

From a75be60c5ed880ed415ebbff5b1245ba0c96c206 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 23 Aug 2022 22:10:03 +0900
Subject: [PATCH 1158/3199] save

---
 scheduler/src/lib.rs | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index f2ba7486f62154..d6e22189a1a961 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -460,8 +460,7 @@ impl Task {
     }
 
     fn mark_as_finished(&self) {
-        assert!(!self.already_finished());
-        assert!(!self.currently_contended());
+        assert!(!self.already_finished() && !self.currently_contended());
         self.uncontended.store(3, std::sync::atomic::Ordering::SeqCst)
     }
 }

From e7192294f32b03771af22beca283163c23750fd8 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 23 Aug 2022 22:12:51 +0900
Subject: [PATCH 1159/3199] save

---
 scheduler/src/lib.rs | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index d6e22189a1a961..31c85b2974c57a 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -558,7 +558,7 @@ pub struct ScheduleStage {}
 
 impl ScheduleStage {
     fn push_to_runnable_queue(
-        (weight, task): (Weight, TaskInQueue),
+        (weight, mut task): (Weight, TaskInQueue),
         runnable_queue: &mut TaskQueue,
         unique_key: &mut u64,
     ) {
@@ -575,6 +575,7 @@ impl ScheduleStage {
         //tx.foo();
         //let unique_weight = (weight << 32) | (*unique_key & 0x0000_0000_ffff_ffff);
         let unique_weight = weight;
+        TaskInQueue::get_mut(&mut task.tx).unwrap();
 
         runnable_queue.add_to_schedule(
             /*

From 6ae9cb99f5ff6f35f456c6f64ce256194ca748cc Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 23 Aug 2022 22:13:07 +0900
Subject: [PATCH 1160/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 31c85b2974c57a..70ebe3dfc7b6cb 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -575,7 +575,7 @@ impl ScheduleStage {
         //tx.foo();
         //let unique_weight = (weight << 32) | (*unique_key & 0x0000_0000_ffff_ffff);
         let unique_weight = weight;
-        TaskInQueue::get_mut(&mut task.tx).unwrap();
+        TaskInQueue::get_mut(&mut task).unwrap();
 
         runnable_queue.add_to_schedule(
             /*

From fc8c46ce4a26489a74eeae01da4fa5e9653a2ee5 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 23 Aug 2022 22:13:20 +0900
Subject: [PATCH 1161/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 70ebe3dfc7b6cb..da4f15d0cc38f0 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -575,7 +575,7 @@ impl ScheduleStage {
         //tx.foo();
         //let unique_weight = (weight << 32) | (*unique_key & 0x0000_0000_ffff_ffff);
         let unique_weight = weight;
-        TaskInQueue::get_mut(&mut task).unwrap();
+        let mut task = TaskInQueue::get_mut(&mut task).unwrap();
 
         runnable_queue.add_to_schedule(
             /*

From 28cac70bb300b322aaef6a50c13638abc7974bfc Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 23 Aug 2022 22:13:32 +0900
Subject: [PATCH 1162/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index da4f15d0cc38f0..af46f84e6f62d4 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -575,7 +575,7 @@ impl ScheduleStage {
         //tx.foo();
         //let unique_weight = (weight << 32) | (*unique_key & 0x0000_0000_ffff_ffff);
         let unique_weight = weight;
-        let mut task = TaskInQueue::get_mut(&mut task).unwrap();
+        let mut unarced_task = TaskInQueue::get_mut(&mut task).unwrap();
 
         runnable_queue.add_to_schedule(
             /*

From 7a98e97d80e7eec2eafc3240727c746e6ef8c0e6 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 23 Aug 2022 22:16:19 +0900
Subject: [PATCH 1163/3199] save

---
 scheduler/src/lib.rs | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index af46f84e6f62d4..439ded35c6feb3 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -489,7 +489,7 @@ impl TaskQueue {
     fn add_to_schedule(&mut self, unique_weight: UniqueWeight, task: TaskInQueue) {
         //trace!("TaskQueue::add(): {:?}", unique_weight);
         let pre_existed = self.tasks.insert(unique_weight, task);
-        debug_assert!(pre_existed.is_none()); //, "identical shouldn't exist: {:?}", unique_weight);
+        assert!(pre_existed.is_none()); //, "identical shouldn't exist: {:?}", unique_weight);
     }
 
     #[inline(never)]
@@ -575,7 +575,6 @@ impl ScheduleStage {
         //tx.foo();
         //let unique_weight = (weight << 32) | (*unique_key & 0x0000_0000_ffff_ffff);
         let unique_weight = weight;
-        let mut unarced_task = TaskInQueue::get_mut(&mut task).unwrap();
 
         runnable_queue.add_to_schedule(
             /*

From 43acdd0c012ef6767872006aaa9b0458a83c220a Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 23 Aug 2022 22:17:43 +0900
Subject: [PATCH 1164/3199] save

---
 scheduler/src/lib.rs | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 439ded35c6feb3..07f16b748e0cf0 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -219,7 +219,7 @@ impl ProvisioningTracker {
         self.remaining_count == 0
     }
 
-    fn tick(&mut self) {
+    fn progress(&mut self) {
         self.remaining_count = self.remaining_count.checked_sub(1).unwrap();
     }
 
@@ -805,13 +805,13 @@ impl ScheduleStage {
                     match address_book.provisioning_trackers.entry(task_id) {
                         std::collections::hash_map::Entry::Occupied(mut tracker_entry) => {
                             let (tracker, task) = tracker_entry.get_mut();
-                            tracker.tick();
+                            tracker.progress();
                             if tracker.is_fulfilled() {
-                                trace!("provisioning tracker tick: {} => {} (!)", tracker.prev_count(), tracker.count());
+                                trace!("provisioning tracker progress: {} => {} (!)", tracker.prev_count(), tracker.count());
                                 let (tracker, task) = tracker_entry.remove();
                                 address_book.fulfilled_provisional_task_ids.insert(task_id, task);
                             } else {
-                                trace!("provisioning tracker tick: {} => {}", tracker.prev_count(), tracker.count());
+                                trace!("provisioning tracker progress: {} => {}", tracker.prev_count(), tracker.count());
                             }
                         },
                         std::collections::hash_map::Entry::Vacant(_) => {

From 1d3464190198a1402bba53cb723ead31d020c1e5 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 23 Aug 2022 22:20:43 +0900
Subject: [PATCH 1165/3199] save

---
 scheduler/src/lib.rs | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 07f16b748e0cf0..145afda902887f 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -875,7 +875,10 @@ impl ScheduleStage {
         weighted_tx: (Weight, TaskInQueue),
         runnable_queue: &mut TaskQueue,
         unique_key: &mut u64,
+        queue_clock: &mut usize,
     ) {
+        weighted_tx.1.observe_clock(queue_clock);
+        *queue_clock = queue_clock.checked_add(1);
         Self::push_to_runnable_queue(weighted_tx, runnable_queue, unique_key)
     }
 
@@ -891,6 +894,7 @@ impl ScheduleStage {
         let mut executing_queue_count = 0;
         let mut current_unique_key = u64::max_value();
         let mut contended_count = 0;
+        let mut queue_clock = 0;
         let (ee_sender, ee_receiver) = crossbeam_channel::unbounded::<Box<ExecutionEnvironment>>();
 
         let (to_next_stage, maybe_jon_handle) = if let Some(to_next_stage) = maybe_to_next_stage {

From 0a04b4242eea8055248b28405a0992d8d49ac91f Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 23 Aug 2022 22:21:01 +0900
Subject: [PATCH 1166/3199] save

---
 scheduler/src/lib.rs | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 145afda902887f..5a21106357c1f0 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -932,14 +932,14 @@ impl ScheduleStage {
                                 to_next_stage.send(processed_execution_environment).unwrap();
                             }
 
-                            Self::register_runnable_task(weighted_tx, runnable_queue, &mut current_unique_key);
+                            Self::register_runnable_task(weighted_tx, runnable_queue, &mut current_unique_key, &mut queue_clock);
 
                             while from.len() > 0 && from_exec.len() == 0 {
                                let i = from.recv().unwrap();
                                 match i {
                                     Multiplexed::FromPrevious(weighted_tx) => {
                                         trace!("recv from previous (after starvation)");
-                                        Self::register_runnable_task(weighted_tx, runnable_queue, &mut current_unique_key);
+                                        Self::register_runnable_task(weighted_tx, runnable_queue, &mut current_unique_key, &mut queue_clock);
                                     }
                                     Multiplexed::FromPreviousBatched(vvv) => {
                                         unreachable!();
@@ -980,7 +980,7 @@ impl ScheduleStage {
                         match i {
                             Multiplexed::FromPrevious(weighted_tx) => {
                                 trace!("recv from previous (after starvation)");
-                                Self::register_runnable_task(weighted_tx, runnable_queue, &mut current_unique_key);
+                                Self::register_runnable_task(weighted_tx, runnable_queue, &mut current_unique_key, &mut queue_clock);
                             }
                             Multiplexed::FromPreviousBatched(vvv) => {
                                 unreachable!();
@@ -1006,7 +1006,7 @@ impl ScheduleStage {
                         match i {
                             Multiplexed::FromPrevious(weighted_tx) => {
                                 trace!("recv from previous (after starvation)");
-                                Self::register_runnable_task(weighted_tx, runnable_queue, &mut current_unique_key);
+                                Self::register_runnable_task(weighted_tx, runnable_queue, &mut current_unique_key, &mut queue_clock);
                             }
                             Multiplexed::FromPreviousBatched(vvv) => {
                                 unreachable!();

From 99c45e17a4ce0d857c25775291488eb530ae5d7b Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 23 Aug 2022 22:21:14 +0900
Subject: [PATCH 1167/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 5a21106357c1f0..3f9e559ee4f8fb 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -878,7 +878,7 @@ impl ScheduleStage {
         queue_clock: &mut usize,
     ) {
         weighted_tx.1.observe_clock(queue_clock);
-        *queue_clock = queue_clock.checked_add(1);
+        *queue_clock = queue_clock.checked_add(1).unwrap();
         Self::push_to_runnable_queue(weighted_tx, runnable_queue, unique_key)
     }
 

From 6156682aea446b08972520c510f4bc8ffc0032c3 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 23 Aug 2022 22:23:03 +0900
Subject: [PATCH 1168/3199] save

---
 scheduler/src/lib.rs | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 3f9e559ee4f8fb..6640ca2a67ccca 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -426,6 +426,7 @@ pub struct Task {
     pub tx: Box<(SanitizedTransaction, Vec<LockAttempt>)>, // actually should be Bundle
     pub contention_count: usize,
     pub uncontended: std::sync::atomic::AtomicUsize,
+    pub queue_time: std::sync::atomic::AtomicUsize,
 }
 
 impl Task {
@@ -433,6 +434,10 @@ impl Task {
         TaskInQueue::new(Self { unique_weight, tx, contention_count: 0, uncontended: Default::default() })
     }
 
+    pub fn observe_clock(&self, clock: usize) {
+        self.queue_time.store(clock, std::sync::atomic::Ordering::SeqCst);
+    }
+
     pub fn clone_for_test(&self) -> Self {
         Self {
             unique_weight: self.unique_weight,

From e0c3f0c73c02079be68d8830adf0abb79a2be3bc Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 23 Aug 2022 22:23:11 +0900
Subject: [PATCH 1169/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 6640ca2a67ccca..d98a2126d281af 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -882,7 +882,7 @@ impl ScheduleStage {
         unique_key: &mut u64,
         queue_clock: &mut usize,
     ) {
-        weighted_tx.1.observe_clock(queue_clock);
+        weighted_tx.1.observe_clock(*queue_clock);
         *queue_clock = queue_clock.checked_add(1).unwrap();
         Self::push_to_runnable_queue(weighted_tx, runnable_queue, unique_key)
     }

From 91c18b45b82a600576cdbf14e6fe94c0c457e146 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 23 Aug 2022 22:23:28 +0900
Subject: [PATCH 1170/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index d98a2126d281af..9fc50d4153c5ab 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -431,7 +431,7 @@ pub struct Task {
 
 impl Task {
     pub fn new_for_queue(unique_weight: UniqueWeight, tx: Box<(SanitizedTransaction, Vec<LockAttempt>)>) -> std::sync::Arc<Self> {
-        TaskInQueue::new(Self { unique_weight, tx, contention_count: 0, uncontended: Default::default() })
+        TaskInQueue::new(Self { unique_weight, tx, contention_count: 0, uncontended: Default::default(), queue_time: Default::default() })
     }
 
     pub fn observe_clock(&self, clock: usize) {

From f17348e2f07cb372491a9f6b9e34b795e980efa9 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 23 Aug 2022 22:24:20 +0900
Subject: [PATCH 1171/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 9fc50d4153c5ab..6021ee15bef6ae 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -431,7 +431,7 @@ pub struct Task {
 
 impl Task {
     pub fn new_for_queue(unique_weight: UniqueWeight, tx: Box<(SanitizedTransaction, Vec<LockAttempt>)>) -> std::sync::Arc<Self> {
-        TaskInQueue::new(Self { unique_weight, tx, contention_count: 0, uncontended: Default::default(), queue_time: Default::default() })
+        TaskInQueue::new(Self { unique_weight, tx, contention_count: 0, uncontended: Default::default(), queue_time: std::sync::atomic::AtomicUsize(usize::max_value()) })
     }
 
     pub fn observe_clock(&self, clock: usize) {

From 8196857623bbcaae7f683d9bb39b2f8889a4b0d7 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 23 Aug 2022 22:24:48 +0900
Subject: [PATCH 1172/3199] save

---
 scheduler/src/lib.rs | 1 +
 1 file changed, 1 insertion(+)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 6021ee15bef6ae..914035e4610b86 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -444,6 +444,7 @@ impl Task {
             tx: Box::new((self.tx.0.clone(), self.tx.1.iter().map(|l| l.clone_for_test()).collect::<Vec<_>>())),
             contention_count: Default::default(),
             uncontended: Default::default(),
+            queue_time: queue_time: std::sync::atomic::AtomicUsize(usize::max_value()),
         }
     }
 

From eb1791c5ecf73ee67fd4f60dc5f2f3477ce098d7 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 23 Aug 2022 22:25:02 +0900
Subject: [PATCH 1173/3199] save

---
 scheduler/src/lib.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 914035e4610b86..a50d04f896a152 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -431,7 +431,7 @@ pub struct Task {
 
 impl Task {
     pub fn new_for_queue(unique_weight: UniqueWeight, tx: Box<(SanitizedTransaction, Vec<LockAttempt>)>) -> std::sync::Arc<Self> {
-        TaskInQueue::new(Self { unique_weight, tx, contention_count: 0, uncontended: Default::default(), queue_time: std::sync::atomic::AtomicUsize(usize::max_value()) })
+        TaskInQueue::new(Self { unique_weight, tx, contention_count: 0, uncontended: Default::default(), queue_time: std::sync::atomic::AtomicUsize::new(usize::max_value()) })
     }
 
     pub fn observe_clock(&self, clock: usize) {
@@ -444,7 +444,7 @@ impl Task {
             tx: Box::new((self.tx.0.clone(), self.tx.1.iter().map(|l| l.clone_for_test()).collect::<Vec<_>>())),
             contention_count: Default::default(),
             uncontended: Default::default(),
-            queue_time: queue_time: std::sync::atomic::AtomicUsize(usize::max_value()),
+            queue_time: queue_time: std::sync::atomic::AtomicUsize::new(usize::max_value()),
         }
     }
 

From 625bde0acd7daefa1ae2b8a0d9bf09731173c5ac Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 23 Aug 2022 22:26:15 +0900
Subject: [PATCH 1174/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index a50d04f896a152..a3a65d8d8a1ebc 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -431,7 +431,7 @@ pub struct Task {
 
 impl Task {
     pub fn new_for_queue(unique_weight: UniqueWeight, tx: Box<(SanitizedTransaction, Vec<LockAttempt>)>) -> std::sync::Arc<Self> {
-        TaskInQueue::new(Self { unique_weight, tx, contention_count: 0, uncontended: Default::default(), queue_time: std::sync::atomic::AtomicUsize::new(usize::max_value()) })
+        TaskInQueue::new(Self { unique_weight, tx, contention_count: 0, uncontended: Default::default(), queue_time: std::sync::atomic::AtomicUsize::new(usize::MAX_VALUE) })
     }
 
     pub fn observe_clock(&self, clock: usize) {

From 0e1917cc71eb850c33dd38dcff82569ea87d9df2 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 23 Aug 2022 22:26:34 +0900
Subject: [PATCH 1175/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index a3a65d8d8a1ebc..84432048d11e05 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -444,7 +444,7 @@ impl Task {
             tx: Box::new((self.tx.0.clone(), self.tx.1.iter().map(|l| l.clone_for_test()).collect::<Vec<_>>())),
             contention_count: Default::default(),
             uncontended: Default::default(),
-            queue_time: queue_time: std::sync::atomic::AtomicUsize::new(usize::max_value()),
+            queue_time: std::sync::atomic::AtomicUsize::new(usize::max_value()),
         }
     }
 

From e201f39b84247a75107c4b2450f18b28dc8a424b Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 23 Aug 2022 22:26:49 +0900
Subject: [PATCH 1176/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 84432048d11e05..3fd3e32ad21d30 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -431,7 +431,7 @@ pub struct Task {
 
 impl Task {
     pub fn new_for_queue(unique_weight: UniqueWeight, tx: Box<(SanitizedTransaction, Vec<LockAttempt>)>) -> std::sync::Arc<Self> {
-        TaskInQueue::new(Self { unique_weight, tx, contention_count: 0, uncontended: Default::default(), queue_time: std::sync::atomic::AtomicUsize::new(usize::MAX_VALUE) })
+        TaskInQueue::new(Self { unique_weight, tx, contention_count: 0, uncontended: Default::default(), queue_time: std::sync::atomic::AtomicUsize::new(usize::max_value()) })
     }
 
     pub fn observe_clock(&self, clock: usize) {

From eee48d490ce12c292e78f5a01d3dc6cc3ff2f94c Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 23 Aug 2022 22:27:55 +0900
Subject: [PATCH 1177/3199] save

---
 scheduler/src/lib.rs | 1 +
 1 file changed, 1 insertion(+)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 3fd3e32ad21d30..676b14fd9abd39 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -909,6 +909,7 @@ impl ScheduleStage {
             let h = std::thread::Builder::new().name("sol-reaper".to_string()).spawn(move || {
                 while let mut a = ee_receiver.recv().unwrap() {
                     assert!(a.task.tx.1.is_empty());
+                    assert!(a.task.queue_time() != usize::max_value());
                     //let lock_attempts = std::mem::take(&mut a.lock_attempts);
                     //drop(lock_attempts);
                     //TaskInQueue::get_mut(&mut a.task).unwrap();

From b0afdc488b926a6c96fb12823e0b0c3e80f87d32 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 23 Aug 2022 22:28:24 +0900
Subject: [PATCH 1178/3199] save

---
 scheduler/src/lib.rs | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 676b14fd9abd39..6614bcc729fdac 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -438,6 +438,11 @@ impl Task {
         self.queue_time.store(clock, std::sync::atomic::Ordering::SeqCst);
     }
 
+    pub fn queue_time(&self) -> usize {
+        self.queue_time.load(std::sync::atomic::Ordering::SeqCst);
+    }
+
+
     pub fn clone_for_test(&self) -> Self {
         Self {
             unique_weight: self.unique_weight,

From 9b0fb71425bf0e7606971fce152330c2cef25e5a Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 23 Aug 2022 22:28:31 +0900
Subject: [PATCH 1179/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 6614bcc729fdac..42e7c4a97197c2 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -439,7 +439,7 @@ impl Task {
     }
 
     pub fn queue_time(&self) -> usize {
-        self.queue_time.load(std::sync::atomic::Ordering::SeqCst);
+        self.queue_time.load(std::sync::atomic::Ordering::SeqCst)
     }
 
 

From c1e12e2229056285c6d156cf424ab5a70681f0f7 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 23 Aug 2022 22:34:00 +0900
Subject: [PATCH 1180/3199] save

---
 scheduler/src/lib.rs | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 42e7c4a97197c2..7e5c24bb66ba21 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -54,7 +54,7 @@ impl PageRc {
         unsafe { MyRcInner::get_mut_unchecked(&mut self.0) }
     }
 
-    pub fn page_ref(&self) -> &Page {
+    fn page_ref(&self) -> &Page {
         //<MyRcInner as std::borrow::Borrow<_>>::borrow(&self.0)
         &*(self.0)
     }
@@ -97,6 +97,10 @@ impl LockAttempt {
             //remembered: false,
         }
     }
+
+    pub fn contended_unique_weights() -> &TaskIds {
+        self.target.page_ref().contended_unique_weights
+    }
 }
 
 type UsageCount = usize;

From f9920d47eaf70a09b3516a7aa8512d49ea6d6da6 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 23 Aug 2022 22:34:09 +0900
Subject: [PATCH 1181/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 7e5c24bb66ba21..7fb2b9de5eb1a1 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -98,7 +98,7 @@ impl LockAttempt {
         }
     }
 
-    pub fn contended_unique_weights() -> &TaskIds {
+    pub fn contended_unique_weights(&self) -> &TaskIds {
         self.target.page_ref().contended_unique_weights
     }
 }

From 7ed46a88980f41353bfc15296aaa62fa8a2955c3 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 23 Aug 2022 22:34:18 +0900
Subject: [PATCH 1182/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 7fb2b9de5eb1a1..965ebe5734019f 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -99,7 +99,7 @@ impl LockAttempt {
     }
 
     pub fn contended_unique_weights(&self) -> &TaskIds {
-        self.target.page_ref().contended_unique_weights
+        &self.target.page_ref().contended_unique_weights
     }
 }
 

From 2f53bbf1baa30753d0dd2509469fbfc43bdc1986 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 23 Aug 2022 22:34:59 +0900
Subject: [PATCH 1183/3199] save

---
 transaction-scheduler-bench/src/main.rs | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/transaction-scheduler-bench/src/main.rs b/transaction-scheduler-bench/src/main.rs
index c72e9f78e0a1e6..f7ec4e55e78672 100644
--- a/transaction-scheduler-bench/src/main.rs
+++ b/transaction-scheduler-bench/src/main.rs
@@ -344,9 +344,9 @@ fn handle_transaction_batch(
 
     let uq = transaction_batch.unique_weight;
     for mut lock_attempt in transaction_batch.lock_attempts.iter_mut() {
-        let page = lock_attempt.target.page_ref();
-        page.contended_unique_weights.remove_task_id(&uq);
-        if let Some(mut task_cursor) = page.contended_unique_weights.heaviest_task_cursor() {
+        let contended_unique_weight = lock_attempt.contended_unique_weights();
+        contended_unique_weights.remove_task_id(&uq);
+        if let Some(mut task_cursor) = contended_unique_weights.heaviest_task_cursor() {
             let mut found = true;
             assert_ne!(task_cursor.key(), &uq);
             while !task_cursor.value().currently_contended() {

From 97104b94ba213a0ec7d28e17314bf9d03c9bdd9e Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 23 Aug 2022 22:35:07 +0900
Subject: [PATCH 1184/3199] save

---
 transaction-scheduler-bench/src/main.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/transaction-scheduler-bench/src/main.rs b/transaction-scheduler-bench/src/main.rs
index f7ec4e55e78672..f7207bbd8a00c3 100644
--- a/transaction-scheduler-bench/src/main.rs
+++ b/transaction-scheduler-bench/src/main.rs
@@ -344,7 +344,7 @@ fn handle_transaction_batch(
 
     let uq = transaction_batch.unique_weight;
     for mut lock_attempt in transaction_batch.lock_attempts.iter_mut() {
-        let contended_unique_weight = lock_attempt.contended_unique_weights();
+        let contended_unique_weights = lock_attempt.contended_unique_weights();
         contended_unique_weights.remove_task_id(&uq);
         if let Some(mut task_cursor) = contended_unique_weights.heaviest_task_cursor() {
             let mut found = true;

From d10b26108a0ef3e1fcc8b5349fe5b36f08d72748 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 23 Aug 2022 22:35:22 +0900
Subject: [PATCH 1185/3199] save

---
 transaction-scheduler-bench/src/main.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/transaction-scheduler-bench/src/main.rs b/transaction-scheduler-bench/src/main.rs
index f7207bbd8a00c3..29d75689bc8181 100644
--- a/transaction-scheduler-bench/src/main.rs
+++ b/transaction-scheduler-bench/src/main.rs
@@ -474,7 +474,7 @@ fn send_packets(
                 let p = (p << 32) | (rng.gen::<u64>() & 0x0000_0000_ffff_ffff);
                 let t = solana_scheduler::Task::new_for_queue(p, v);
                 for lock_attempt in t.tx.1.iter() {
-                    lock_attempt.target.page_ref().contended_unique_weights.insert_task_id(p, solana_scheduler::TaskInQueue::clone(&t));
+                    lock_attempt.contended_unique_weight()s.insert_task_id(p, solana_scheduler::TaskInQueue::clone(&t));
                 }
                 packet_batch_sender.send(solana_scheduler::Multiplexed::FromPrevious((p, t))).unwrap();
             }

From 53525806d39d064138e04e0a0fd6ff003bc83e64 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 23 Aug 2022 22:35:29 +0900
Subject: [PATCH 1186/3199] save

---
 transaction-scheduler-bench/src/main.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/transaction-scheduler-bench/src/main.rs b/transaction-scheduler-bench/src/main.rs
index 29d75689bc8181..b7d744ddce717a 100644
--- a/transaction-scheduler-bench/src/main.rs
+++ b/transaction-scheduler-bench/src/main.rs
@@ -474,7 +474,7 @@ fn send_packets(
                 let p = (p << 32) | (rng.gen::<u64>() & 0x0000_0000_ffff_ffff);
                 let t = solana_scheduler::Task::new_for_queue(p, v);
                 for lock_attempt in t.tx.1.iter() {
-                    lock_attempt.contended_unique_weight()s.insert_task_id(p, solana_scheduler::TaskInQueue::clone(&t));
+                    lock_attempt.contended_unique_weights().insert_task_id(p, solana_scheduler::TaskInQueue::clone(&t));
                 }
                 packet_batch_sender.send(solana_scheduler::Multiplexed::FromPrevious((p, t))).unwrap();
             }

From 0adde0caf5729b8ebb98e23b4cc53bfdafd4fc1a Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 23 Aug 2022 22:36:11 +0900
Subject: [PATCH 1187/3199] save

---
 transaction-scheduler-bench/src/main.rs | 1 +
 1 file changed, 1 insertion(+)

diff --git a/transaction-scheduler-bench/src/main.rs b/transaction-scheduler-bench/src/main.rs
index b7d744ddce717a..932b1753548d03 100644
--- a/transaction-scheduler-bench/src/main.rs
+++ b/transaction-scheduler-bench/src/main.rs
@@ -359,6 +359,7 @@ fn handle_transaction_batch(
                     break;
                 }
             }
+            drop(contended_unique_weights);
             if found {
                 lock_attempt.heaviest_uncontended = Some(solana_scheduler::TaskInQueue::clone(task_cursor.value()));
             }

From 9e6110967a082bbd319593bdb86d9fb616c89942 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 23 Aug 2022 22:36:33 +0900
Subject: [PATCH 1188/3199] save

---
 transaction-scheduler-bench/src/main.rs | 1 +
 1 file changed, 1 insertion(+)

diff --git a/transaction-scheduler-bench/src/main.rs b/transaction-scheduler-bench/src/main.rs
index 932b1753548d03..2dc43d524eb569 100644
--- a/transaction-scheduler-bench/src/main.rs
+++ b/transaction-scheduler-bench/src/main.rs
@@ -359,6 +359,7 @@ fn handle_transaction_batch(
                     break;
                 }
             }
+            drop(task_cursor);
             drop(contended_unique_weights);
             if found {
                 lock_attempt.heaviest_uncontended = Some(solana_scheduler::TaskInQueue::clone(task_cursor.value()));

From 3c7ef1dde24f603417a374784ece37d1d33217ad Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 23 Aug 2022 22:37:11 +0900
Subject: [PATCH 1189/3199] save

---
 transaction-scheduler-bench/src/main.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/transaction-scheduler-bench/src/main.rs b/transaction-scheduler-bench/src/main.rs
index 2dc43d524eb569..a7e3fd6bc9e233 100644
--- a/transaction-scheduler-bench/src/main.rs
+++ b/transaction-scheduler-bench/src/main.rs
@@ -359,8 +359,8 @@ fn handle_transaction_batch(
                     break;
                 }
             }
-            drop(task_cursor);
             drop(contended_unique_weights);
+            drop(task_cursor);
             if found {
                 lock_attempt.heaviest_uncontended = Some(solana_scheduler::TaskInQueue::clone(task_cursor.value()));
             }

From 68c15bb27000d16d38b6ab680b10677e2126a90d Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 23 Aug 2022 22:40:48 +0900
Subject: [PATCH 1190/3199] save

---
 transaction-scheduler-bench/src/main.rs | 11 +++++------
 1 file changed, 5 insertions(+), 6 deletions(-)

diff --git a/transaction-scheduler-bench/src/main.rs b/transaction-scheduler-bench/src/main.rs
index a7e3fd6bc9e233..90597d02a9561f 100644
--- a/transaction-scheduler-bench/src/main.rs
+++ b/transaction-scheduler-bench/src/main.rs
@@ -346,7 +346,7 @@ fn handle_transaction_batch(
     for mut lock_attempt in transaction_batch.lock_attempts.iter_mut() {
         let contended_unique_weights = lock_attempt.contended_unique_weights();
         contended_unique_weights.remove_task_id(&uq);
-        if let Some(mut task_cursor) = contended_unique_weights.heaviest_task_cursor() {
+        let maybe_task = if let Some(mut task_cursor) = contended_unique_weights.heaviest_task_cursor() {
             let mut found = true;
             assert_ne!(task_cursor.key(), &uq);
             while !task_cursor.value().currently_contended() {
@@ -359,11 +359,10 @@ fn handle_transaction_batch(
                     break;
                 }
             }
-            drop(contended_unique_weights);
-            drop(task_cursor);
-            if found {
-                lock_attempt.heaviest_uncontended = Some(solana_scheduler::TaskInQueue::clone(task_cursor.value()));
-            }
+            found.and_then(Some(solana_scheduler::TaskInQueue::clone(task_cursor.value())));
+        };
+        if let Some(task) = maybe_task {
+            lock_attempt.heaviest_uncontended = Some(task);
         }
     }
     completed_transaction_sender.0

From a31f95ef230d2c055009e0d0918b1e1f1c5e4a65 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 23 Aug 2022 22:40:57 +0900
Subject: [PATCH 1191/3199] save

---
 transaction-scheduler-bench/src/main.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/transaction-scheduler-bench/src/main.rs b/transaction-scheduler-bench/src/main.rs
index 90597d02a9561f..fc2389e0dc9d6e 100644
--- a/transaction-scheduler-bench/src/main.rs
+++ b/transaction-scheduler-bench/src/main.rs
@@ -359,7 +359,7 @@ fn handle_transaction_batch(
                     break;
                 }
             }
-            found.and_then(Some(solana_scheduler::TaskInQueue::clone(task_cursor.value())));
+            found.then(Some(solana_scheduler::TaskInQueue::clone(task_cursor.value())));
         };
         if let Some(task) = maybe_task {
             lock_attempt.heaviest_uncontended = Some(task);

From 99a10952d458ea23e336169caa2760c9dc337e4b Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 23 Aug 2022 22:41:08 +0900
Subject: [PATCH 1192/3199] save

---
 transaction-scheduler-bench/src/main.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/transaction-scheduler-bench/src/main.rs b/transaction-scheduler-bench/src/main.rs
index fc2389e0dc9d6e..98e87a0cf4894a 100644
--- a/transaction-scheduler-bench/src/main.rs
+++ b/transaction-scheduler-bench/src/main.rs
@@ -359,7 +359,7 @@ fn handle_transaction_batch(
                     break;
                 }
             }
-            found.then(Some(solana_scheduler::TaskInQueue::clone(task_cursor.value())));
+            found.then_some(solana_scheduler::TaskInQueue::clone(task_cursor.value()));
         };
         if let Some(task) = maybe_task {
             lock_attempt.heaviest_uncontended = Some(task);

From 8872a0c0e8ae209f29014b6684ba9198bb2d5cb8 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 23 Aug 2022 22:41:21 +0900
Subject: [PATCH 1193/3199] save

---
 transaction-scheduler-bench/src/main.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/transaction-scheduler-bench/src/main.rs b/transaction-scheduler-bench/src/main.rs
index 98e87a0cf4894a..ba83790758a396 100644
--- a/transaction-scheduler-bench/src/main.rs
+++ b/transaction-scheduler-bench/src/main.rs
@@ -359,7 +359,7 @@ fn handle_transaction_batch(
                     break;
                 }
             }
-            found.then_some(solana_scheduler::TaskInQueue::clone(task_cursor.value()));
+            found.then_some(solana_scheduler::TaskInQueue::clone(task_cursor.value()))
         };
         if let Some(task) = maybe_task {
             lock_attempt.heaviest_uncontended = Some(task);

From fa70a1c333d5890a9fb97cbeabcce8e46fb4ea8f Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 23 Aug 2022 22:42:05 +0900
Subject: [PATCH 1194/3199] save

---
 transaction-scheduler-bench/src/main.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/transaction-scheduler-bench/src/main.rs b/transaction-scheduler-bench/src/main.rs
index ba83790758a396..93f12f409793e9 100644
--- a/transaction-scheduler-bench/src/main.rs
+++ b/transaction-scheduler-bench/src/main.rs
@@ -346,7 +346,7 @@ fn handle_transaction_batch(
     for mut lock_attempt in transaction_batch.lock_attempts.iter_mut() {
         let contended_unique_weights = lock_attempt.contended_unique_weights();
         contended_unique_weights.remove_task_id(&uq);
-        let maybe_task = if let Some(mut task_cursor) = contended_unique_weights.heaviest_task_cursor() {
+        let maybe_task = contended_unique_weights.heaviest_task_cursor().map(|mut task_cursor| {
             let mut found = true;
             assert_ne!(task_cursor.key(), &uq);
             while !task_cursor.value().currently_contended() {
@@ -360,7 +360,7 @@ fn handle_transaction_batch(
                 }
             }
             found.then_some(solana_scheduler::TaskInQueue::clone(task_cursor.value()))
-        };
+        });
         if let Some(task) = maybe_task {
             lock_attempt.heaviest_uncontended = Some(task);
         }

From 1583218cf6f0ed2c92f5b8920975048fac7333d0 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 23 Aug 2022 22:42:59 +0900
Subject: [PATCH 1195/3199] save

---
 transaction-scheduler-bench/src/main.rs | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/transaction-scheduler-bench/src/main.rs b/transaction-scheduler-bench/src/main.rs
index 93f12f409793e9..c376255f51b3c6 100644
--- a/transaction-scheduler-bench/src/main.rs
+++ b/transaction-scheduler-bench/src/main.rs
@@ -360,10 +360,9 @@ fn handle_transaction_batch(
                 }
             }
             found.then_some(solana_scheduler::TaskInQueue::clone(task_cursor.value()))
-        });
-        if let Some(task) = maybe_task {
+        }).and_then(|task| {
             lock_attempt.heaviest_uncontended = Some(task);
-        }
+        });
     }
     completed_transaction_sender.0
         .send(transaction_batch)

From aeb443bd0eefc0d329f349e7a2e0f5cdae7a2e30 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 23 Aug 2022 22:43:26 +0900
Subject: [PATCH 1196/3199] save

---
 transaction-scheduler-bench/src/main.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/transaction-scheduler-bench/src/main.rs b/transaction-scheduler-bench/src/main.rs
index c376255f51b3c6..a339d808a4f7a1 100644
--- a/transaction-scheduler-bench/src/main.rs
+++ b/transaction-scheduler-bench/src/main.rs
@@ -361,7 +361,7 @@ fn handle_transaction_batch(
             }
             found.then_some(solana_scheduler::TaskInQueue::clone(task_cursor.value()))
         }).and_then(|task| {
-            lock_attempt.heaviest_uncontended = Some(task);
+            lock_attempt.heaviest_uncontended = Some(task)
         });
     }
     completed_transaction_sender.0

From 2f6c581cf940ac7fff14fd1ff0cc233c091eb35e Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 23 Aug 2022 22:43:44 +0900
Subject: [PATCH 1197/3199] save

---
 transaction-scheduler-bench/src/main.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/transaction-scheduler-bench/src/main.rs b/transaction-scheduler-bench/src/main.rs
index a339d808a4f7a1..217151821ddc34 100644
--- a/transaction-scheduler-bench/src/main.rs
+++ b/transaction-scheduler-bench/src/main.rs
@@ -361,7 +361,7 @@ fn handle_transaction_batch(
             }
             found.then_some(solana_scheduler::TaskInQueue::clone(task_cursor.value()))
         }).and_then(|task| {
-            lock_attempt.heaviest_uncontended = Some(task)
+            lock_attempt.heaviest_uncontended = task
         });
     }
     completed_transaction_sender.0

From 3adbea60d286bff30e8f9b319001c14ea5d11778 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 23 Aug 2022 22:44:15 +0900
Subject: [PATCH 1198/3199] save

---
 transaction-scheduler-bench/src/main.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/transaction-scheduler-bench/src/main.rs b/transaction-scheduler-bench/src/main.rs
index 217151821ddc34..0e91d04c1a1a96 100644
--- a/transaction-scheduler-bench/src/main.rs
+++ b/transaction-scheduler-bench/src/main.rs
@@ -360,8 +360,8 @@ fn handle_transaction_batch(
                 }
             }
             found.then_some(solana_scheduler::TaskInQueue::clone(task_cursor.value()))
-        }).and_then(|task| {
-            lock_attempt.heaviest_uncontended = task
+        }).then(|task| {
+            lock_attempt.heaviest_uncontended = task;
         });
     }
     completed_transaction_sender.0

From b698f3d7b7154157c3c00f231d190ebbe2b8624d Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 23 Aug 2022 22:44:23 +0900
Subject: [PATCH 1199/3199] save

---
 transaction-scheduler-bench/src/main.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/transaction-scheduler-bench/src/main.rs b/transaction-scheduler-bench/src/main.rs
index 0e91d04c1a1a96..7d07f5234e7d91 100644
--- a/transaction-scheduler-bench/src/main.rs
+++ b/transaction-scheduler-bench/src/main.rs
@@ -360,7 +360,7 @@ fn handle_transaction_batch(
                 }
             }
             found.then_some(solana_scheduler::TaskInQueue::clone(task_cursor.value()))
-        }).then(|task| {
+        }).and(|task| {
             lock_attempt.heaviest_uncontended = task;
         });
     }

From 0a08655cfa518c5ff40413c09682556516e5b279 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 23 Aug 2022 22:44:43 +0900
Subject: [PATCH 1200/3199] save

---
 transaction-scheduler-bench/src/main.rs | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/transaction-scheduler-bench/src/main.rs b/transaction-scheduler-bench/src/main.rs
index 7d07f5234e7d91..9fa318047f7a61 100644
--- a/transaction-scheduler-bench/src/main.rs
+++ b/transaction-scheduler-bench/src/main.rs
@@ -360,8 +360,9 @@ fn handle_transaction_batch(
                 }
             }
             found.then_some(solana_scheduler::TaskInQueue::clone(task_cursor.value()))
-        }).and(|task| {
+        }).map(|task| {
             lock_attempt.heaviest_uncontended = task;
+            ()
         });
     }
     completed_transaction_sender.0

From f4443217c7b973a7dc0cfb2598c0b165fabd9fb2 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 23 Aug 2022 22:45:36 +0900
Subject: [PATCH 1201/3199] save

---
 transaction-scheduler-bench/src/main.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/transaction-scheduler-bench/src/main.rs b/transaction-scheduler-bench/src/main.rs
index 9fa318047f7a61..8dba99fcb531b3 100644
--- a/transaction-scheduler-bench/src/main.rs
+++ b/transaction-scheduler-bench/src/main.rs
@@ -360,7 +360,7 @@ fn handle_transaction_batch(
                 }
             }
             found.then_some(solana_scheduler::TaskInQueue::clone(task_cursor.value()))
-        }).map(|task| {
+        }).map(|task: usize| {
             lock_attempt.heaviest_uncontended = task;
             ()
         });

From 8d6cf9163d506ce196bf3ece528bf3e88bccd9d3 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 23 Aug 2022 22:46:20 +0900
Subject: [PATCH 1202/3199] save

---
 transaction-scheduler-bench/src/main.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/transaction-scheduler-bench/src/main.rs b/transaction-scheduler-bench/src/main.rs
index 8dba99fcb531b3..a7c482ed5ba03c 100644
--- a/transaction-scheduler-bench/src/main.rs
+++ b/transaction-scheduler-bench/src/main.rs
@@ -346,7 +346,7 @@ fn handle_transaction_batch(
     for mut lock_attempt in transaction_batch.lock_attempts.iter_mut() {
         let contended_unique_weights = lock_attempt.contended_unique_weights();
         contended_unique_weights.remove_task_id(&uq);
-        let maybe_task = contended_unique_weights.heaviest_task_cursor().map(|mut task_cursor| {
+        contended_unique_weights.heaviest_task_cursor().map(|mut task_cursor| {
             let mut found = true;
             assert_ne!(task_cursor.key(), &uq);
             while !task_cursor.value().currently_contended() {
@@ -360,7 +360,7 @@ fn handle_transaction_batch(
                 }
             }
             found.then_some(solana_scheduler::TaskInQueue::clone(task_cursor.value()))
-        }).map(|task: usize| {
+        }).map(|task| {
             lock_attempt.heaviest_uncontended = task;
             ()
         });

From 0758b89acd004a93ba0384bda11a09e1698508bf Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 23 Aug 2022 22:48:21 +0900
Subject: [PATCH 1203/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 965ebe5734019f..6c47bbfc477eb8 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -69,7 +69,7 @@ enum LockStatus {
 
 #[derive(Debug)]
 pub struct LockAttempt {
-    pub target: PageRc,
+    target: PageRc,
     status: LockStatus,
     requested_usage: RequestedUsage,
     //pub heaviest_uncontended: arc_swap::ArcSwapOption<Task>,

From 89714dbfe8757403eef1054d6a186b30a9b2cd0a Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 23 Aug 2022 22:53:43 +0900
Subject: [PATCH 1204/3199] save

---
 transaction-scheduler-bench/src/main.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/transaction-scheduler-bench/src/main.rs b/transaction-scheduler-bench/src/main.rs
index a7c482ed5ba03c..cd42510661cb93 100644
--- a/transaction-scheduler-bench/src/main.rs
+++ b/transaction-scheduler-bench/src/main.rs
@@ -359,7 +359,7 @@ fn handle_transaction_batch(
                     break;
                 }
             }
-            found.then_some(solana_scheduler::TaskInQueue::clone(task_cursor.value()))
+            found.then_else(solana_scheduler::TaskInQueue::clone(task_cursor.value()))
         }).map(|task| {
             lock_attempt.heaviest_uncontended = task;
             ()

From 24d8a1fdb7bf27ceec65eade1b11f052936b5aaa Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 23 Aug 2022 22:54:00 +0900
Subject: [PATCH 1205/3199] save

---
 transaction-scheduler-bench/src/main.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/transaction-scheduler-bench/src/main.rs b/transaction-scheduler-bench/src/main.rs
index cd42510661cb93..bdf2931a8cda39 100644
--- a/transaction-scheduler-bench/src/main.rs
+++ b/transaction-scheduler-bench/src/main.rs
@@ -359,7 +359,7 @@ fn handle_transaction_batch(
                     break;
                 }
             }
-            found.then_else(solana_scheduler::TaskInQueue::clone(task_cursor.value()))
+            found.then(solana_scheduler::TaskInQueue::clone(task_cursor.value()))
         }).map(|task| {
             lock_attempt.heaviest_uncontended = task;
             ()

From 8a974fb25bab49719f79edbd50b727868cf8871d Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 23 Aug 2022 22:54:09 +0900
Subject: [PATCH 1206/3199] save

---
 transaction-scheduler-bench/src/main.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/transaction-scheduler-bench/src/main.rs b/transaction-scheduler-bench/src/main.rs
index bdf2931a8cda39..10d5f7e8b33759 100644
--- a/transaction-scheduler-bench/src/main.rs
+++ b/transaction-scheduler-bench/src/main.rs
@@ -359,7 +359,7 @@ fn handle_transaction_batch(
                     break;
                 }
             }
-            found.then(solana_scheduler::TaskInQueue::clone(task_cursor.value()))
+            found.then(|| solana_scheduler::TaskInQueue::clone(task_cursor.value()))
         }).map(|task| {
             lock_attempt.heaviest_uncontended = task;
             ()

From 2ac22c69fefac7870a30ec493acb44faa5456017 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Wed, 24 Aug 2022 13:52:09 +0900
Subject: [PATCH 1207/3199] save

---
 transaction-scheduler-bench/src/main.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/transaction-scheduler-bench/src/main.rs b/transaction-scheduler-bench/src/main.rs
index 10d5f7e8b33759..e94679821e136d 100644
--- a/transaction-scheduler-bench/src/main.rs
+++ b/transaction-scheduler-bench/src/main.rs
@@ -360,8 +360,8 @@ fn handle_transaction_batch(
                 }
             }
             found.then(|| solana_scheduler::TaskInQueue::clone(task_cursor.value()))
-        }).map(|task| {
-            lock_attempt.heaviest_uncontended = task;
+        }).flatten().map(|task| {
+            lock_attempt.heaviest_uncontended = Some(task);
             ()
         });
     }

From 0dd5eda788cad56d5f4fc96b42542ef3ac5e55b9 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Wed, 24 Aug 2022 13:53:30 +0900
Subject: [PATCH 1208/3199] save

---
 transaction-scheduler-bench/src/main.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/transaction-scheduler-bench/src/main.rs b/transaction-scheduler-bench/src/main.rs
index e94679821e136d..634c3c2c476979 100644
--- a/transaction-scheduler-bench/src/main.rs
+++ b/transaction-scheduler-bench/src/main.rs
@@ -363,7 +363,7 @@ fn handle_transaction_batch(
         }).flatten().map(|task| {
             lock_attempt.heaviest_uncontended = Some(task);
             ()
-        });
+        })
     }
     completed_transaction_sender.0
         .send(transaction_batch)

From 8ba4b7e97a401d51b07bcc8ce3bca0ea16924a89 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Wed, 24 Aug 2022 13:54:05 +0900
Subject: [PATCH 1209/3199] save

---
 transaction-scheduler-bench/src/main.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/transaction-scheduler-bench/src/main.rs b/transaction-scheduler-bench/src/main.rs
index 634c3c2c476979..a06601f8819037 100644
--- a/transaction-scheduler-bench/src/main.rs
+++ b/transaction-scheduler-bench/src/main.rs
@@ -360,10 +360,10 @@ fn handle_transaction_batch(
                 }
             }
             found.then(|| solana_scheduler::TaskInQueue::clone(task_cursor.value()))
-        }).flatten().map(|task| {
+        }).flatten().then(|task| {
             lock_attempt.heaviest_uncontended = Some(task);
             ()
-        })
+        });
     }
     completed_transaction_sender.0
         .send(transaction_batch)

From c9b806054d1d92368a0ca948a1898da9729fa9b1 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Wed, 24 Aug 2022 13:54:20 +0900
Subject: [PATCH 1210/3199] save

---
 transaction-scheduler-bench/src/main.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/transaction-scheduler-bench/src/main.rs b/transaction-scheduler-bench/src/main.rs
index a06601f8819037..e94679821e136d 100644
--- a/transaction-scheduler-bench/src/main.rs
+++ b/transaction-scheduler-bench/src/main.rs
@@ -360,7 +360,7 @@ fn handle_transaction_batch(
                 }
             }
             found.then(|| solana_scheduler::TaskInQueue::clone(task_cursor.value()))
-        }).flatten().then(|task| {
+        }).flatten().map(|task| {
             lock_attempt.heaviest_uncontended = Some(task);
             ()
         });

From ddae797a201ac0c4350e8ab046e470f2d277ce4b Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Wed, 24 Aug 2022 13:59:49 +0900
Subject: [PATCH 1211/3199] save

---
 scheduler/src/lib.rs | 19 ++++++++++++-------
 1 file changed, 12 insertions(+), 7 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 6c47bbfc477eb8..4d519eed3ddc0d 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -430,20 +430,25 @@ pub struct Task {
     pub tx: Box<(SanitizedTransaction, Vec<LockAttempt>)>, // actually should be Bundle
     pub contention_count: usize,
     pub uncontended: std::sync::atomic::AtomicUsize,
-    pub queue_time: std::sync::atomic::AtomicUsize,
+    pub sequence_time: std::sync::atomic::AtomicUsize,
 }
 
+// sequence_time
+// queue_time
+// execute_time
+// terminate_time
+
 impl Task {
     pub fn new_for_queue(unique_weight: UniqueWeight, tx: Box<(SanitizedTransaction, Vec<LockAttempt>)>) -> std::sync::Arc<Self> {
-        TaskInQueue::new(Self { unique_weight, tx, contention_count: 0, uncontended: Default::default(), queue_time: std::sync::atomic::AtomicUsize::new(usize::max_value()) })
+        TaskInQueue::new(Self { unique_weight, tx, contention_count: 0, uncontended: Default::default(), sequence_time: std::sync::atomic::AtomicUsize::new(usize::max_value()) })
     }
 
     pub fn observe_clock(&self, clock: usize) {
-        self.queue_time.store(clock, std::sync::atomic::Ordering::SeqCst);
+        self.sequence_time.store(clock, std::sync::atomic::Ordering::SeqCst);
     }
 
-    pub fn queue_time(&self) -> usize {
-        self.queue_time.load(std::sync::atomic::Ordering::SeqCst)
+    pub fn sequence_time(&self) -> usize {
+        self.sequence_time.load(std::sync::atomic::Ordering::SeqCst)
     }
 
 
@@ -453,7 +458,7 @@ impl Task {
             tx: Box::new((self.tx.0.clone(), self.tx.1.iter().map(|l| l.clone_for_test()).collect::<Vec<_>>())),
             contention_count: Default::default(),
             uncontended: Default::default(),
-            queue_time: std::sync::atomic::AtomicUsize::new(usize::max_value()),
+            sequence_time: std::sync::atomic::AtomicUsize::new(usize::max_value()),
         }
     }
 
@@ -918,7 +923,7 @@ impl ScheduleStage {
             let h = std::thread::Builder::new().name("sol-reaper".to_string()).spawn(move || {
                 while let mut a = ee_receiver.recv().unwrap() {
                     assert!(a.task.tx.1.is_empty());
-                    assert!(a.task.queue_time() != usize::max_value());
+                    assert!(a.task.sequence_time() != usize::max_value());
                     //let lock_attempts = std::mem::take(&mut a.lock_attempts);
                     //drop(lock_attempts);
                     //TaskInQueue::get_mut(&mut a.task).unwrap();

From 5a30de72c7c97d5a04192ea692a910c38f059c35 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Wed, 24 Aug 2022 14:05:25 +0900
Subject: [PATCH 1212/3199] save

---
 transaction-scheduler-bench/src/main.rs | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/transaction-scheduler-bench/src/main.rs b/transaction-scheduler-bench/src/main.rs
index e94679821e136d..f909722efecaab 100644
--- a/transaction-scheduler-bench/src/main.rs
+++ b/transaction-scheduler-bench/src/main.rs
@@ -349,17 +349,19 @@ fn handle_transaction_batch(
         contended_unique_weights.heaviest_task_cursor().map(|mut task_cursor| {
             let mut found = true;
             assert_ne!(task_cursor.key(), &uq);
-            while !task_cursor.value().currently_contended() {
+            let mut value = task_cursor.value();
+            while !value.currently_contended() {
                 if let Some(new_cursor) = task_cursor.prev() {
                     assert!(new_cursor.key() < task_cursor.key());
                     assert_ne!(new_cursor.key(), &uq);
                     task_cursor = new_cursor;
+                    value = task_cursor.value();
                 } else {
                     found = false;
                     break;
                 }
             }
-            found.then(|| solana_scheduler::TaskInQueue::clone(task_cursor.value()))
+            found.then(|| solana_scheduler::TaskInQueue::clone(value))
         }).flatten().map(|task| {
             lock_attempt.heaviest_uncontended = Some(task);
             ()

From 231f9439518389f08f5cd7290467f0aef96cc1f5 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Wed, 24 Aug 2022 14:05:51 +0900
Subject: [PATCH 1213/3199] save

---
 transaction-scheduler-bench/src/main.rs | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/transaction-scheduler-bench/src/main.rs b/transaction-scheduler-bench/src/main.rs
index f909722efecaab..b0bf410cab9757 100644
--- a/transaction-scheduler-bench/src/main.rs
+++ b/transaction-scheduler-bench/src/main.rs
@@ -349,19 +349,19 @@ fn handle_transaction_batch(
         contended_unique_weights.heaviest_task_cursor().map(|mut task_cursor| {
             let mut found = true;
             assert_ne!(task_cursor.key(), &uq);
-            let mut value = task_cursor.value();
-            while !value.currently_contended() {
+            let mut task = task_cursor.value();
+            while !task.currently_contended() {
                 if let Some(new_cursor) = task_cursor.prev() {
                     assert!(new_cursor.key() < task_cursor.key());
                     assert_ne!(new_cursor.key(), &uq);
                     task_cursor = new_cursor;
-                    value = task_cursor.value();
+                    task = task_cursor.value();
                 } else {
                     found = false;
                     break;
                 }
             }
-            found.then(|| solana_scheduler::TaskInQueue::clone(value))
+            found.then(|| solana_scheduler::TaskInQueue::clone(task))
         }).flatten().map(|task| {
             lock_attempt.heaviest_uncontended = Some(task);
             ()

From 4e9c03b19f10410a1005769b7238af0067002de9 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Wed, 24 Aug 2022 14:06:50 +0900
Subject: [PATCH 1214/3199] save

---
 transaction-scheduler-bench/src/main.rs | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/transaction-scheduler-bench/src/main.rs b/transaction-scheduler-bench/src/main.rs
index b0bf410cab9757..b3e601b9151802 100644
--- a/transaction-scheduler-bench/src/main.rs
+++ b/transaction-scheduler-bench/src/main.rs
@@ -354,8 +354,7 @@ fn handle_transaction_batch(
                 if let Some(new_cursor) = task_cursor.prev() {
                     assert!(new_cursor.key() < task_cursor.key());
                     assert_ne!(new_cursor.key(), &uq);
-                    task_cursor = new_cursor;
-                    task = task_cursor.value();
+                    (task_cursor, task) = (new_cursor, new_cursor.value());
                 } else {
                     found = false;
                     break;

From b774b8b864871589e68cb658d9a350715e907834 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Wed, 24 Aug 2022 14:07:20 +0900
Subject: [PATCH 1215/3199] save

---
 transaction-scheduler-bench/src/main.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/transaction-scheduler-bench/src/main.rs b/transaction-scheduler-bench/src/main.rs
index b3e601b9151802..98dc54dc9a50d9 100644
--- a/transaction-scheduler-bench/src/main.rs
+++ b/transaction-scheduler-bench/src/main.rs
@@ -354,7 +354,7 @@ fn handle_transaction_batch(
                 if let Some(new_cursor) = task_cursor.prev() {
                     assert!(new_cursor.key() < task_cursor.key());
                     assert_ne!(new_cursor.key(), &uq);
-                    (task_cursor, task) = (new_cursor, new_cursor.value());
+                    (task, task_cursor) = (new_cursor.value(), new_cursor);
                 } else {
                     found = false;
                     break;

From 88cd7c7f9f16a97413d845ab9ba322a677d75e8f Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Wed, 24 Aug 2022 14:07:33 +0900
Subject: [PATCH 1216/3199] save

---
 transaction-scheduler-bench/src/main.rs | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/transaction-scheduler-bench/src/main.rs b/transaction-scheduler-bench/src/main.rs
index 98dc54dc9a50d9..b0bf410cab9757 100644
--- a/transaction-scheduler-bench/src/main.rs
+++ b/transaction-scheduler-bench/src/main.rs
@@ -354,7 +354,8 @@ fn handle_transaction_batch(
                 if let Some(new_cursor) = task_cursor.prev() {
                     assert!(new_cursor.key() < task_cursor.key());
                     assert_ne!(new_cursor.key(), &uq);
-                    (task, task_cursor) = (new_cursor.value(), new_cursor);
+                    task_cursor = new_cursor;
+                    task = task_cursor.value();
                 } else {
                     found = false;
                     break;

From c076245ff760eafa1a0e7e651c4cd6574a0cb9de Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Wed, 24 Aug 2022 14:09:38 +0900
Subject: [PATCH 1217/3199] save

---
 scheduler/src/lib.rs                    | 9 ++-------
 transaction-scheduler-bench/src/main.rs | 2 +-
 2 files changed, 3 insertions(+), 8 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 4d519eed3ddc0d..400141cc7a5a18 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -141,7 +141,7 @@ pub struct TaskIds {
 
 impl TaskIds {
     #[inline(never)]
-    pub fn insert_task_id(&self, u: TaskId, task: TaskInQueue) {
+    pub fn insert_task(&self, u: TaskId, task: TaskInQueue) {
         let mut is_inserted = false;
         self.task_ids.get_or_insert_with(u, || {
             is_inserted = true;
@@ -151,16 +151,11 @@ impl TaskIds {
     }
 
     #[inline(never)]
-    pub fn remove_task_id(&self, u: &TaskId) {
+    pub fn remove_task(&self, u: &TaskId) {
         let removed_entry = self.task_ids.remove(u);
         assert!(removed_entry.is_some());
     }
 
-    #[inline(never)]
-    fn heaviest_task_id(&self) -> Option<TaskId> {
-        self.task_ids.back().map(|e| *(e.key()))
-    }
-
     #[inline(never)]
     pub fn heaviest_task_cursor(&self) -> Option<crossbeam_skiplist::map::Entry<'_, UniqueWeight, TaskInQueue>> {
         self.task_ids.back()
diff --git a/transaction-scheduler-bench/src/main.rs b/transaction-scheduler-bench/src/main.rs
index b0bf410cab9757..03a9df788e876f 100644
--- a/transaction-scheduler-bench/src/main.rs
+++ b/transaction-scheduler-bench/src/main.rs
@@ -477,7 +477,7 @@ fn send_packets(
                 let p = (p << 32) | (rng.gen::<u64>() & 0x0000_0000_ffff_ffff);
                 let t = solana_scheduler::Task::new_for_queue(p, v);
                 for lock_attempt in t.tx.1.iter() {
-                    lock_attempt.contended_unique_weights().insert_task_id(p, solana_scheduler::TaskInQueue::clone(&t));
+                    lock_attempt.contended_unique_weights().insert_task(p, solana_scheduler::TaskInQueue::clone(&t));
                 }
                 packet_batch_sender.send(solana_scheduler::Multiplexed::FromPrevious((p, t))).unwrap();
             }

From 9cd780d5d2b30621008cdc435c28cb1cc7c6833b Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Wed, 24 Aug 2022 14:09:58 +0900
Subject: [PATCH 1218/3199] save

---
 transaction-scheduler-bench/src/main.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/transaction-scheduler-bench/src/main.rs b/transaction-scheduler-bench/src/main.rs
index 03a9df788e876f..8c6ea01f010db6 100644
--- a/transaction-scheduler-bench/src/main.rs
+++ b/transaction-scheduler-bench/src/main.rs
@@ -345,7 +345,7 @@ fn handle_transaction_batch(
     let uq = transaction_batch.unique_weight;
     for mut lock_attempt in transaction_batch.lock_attempts.iter_mut() {
         let contended_unique_weights = lock_attempt.contended_unique_weights();
-        contended_unique_weights.remove_task_id(&uq);
+        contended_unique_weights.remove_task(&uq);
         contended_unique_weights.heaviest_task_cursor().map(|mut task_cursor| {
             let mut found = true;
             assert_ne!(task_cursor.key(), &uq);

From 2299162788c7441560321094a938a7756a0a061a Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Wed, 24 Aug 2022 14:15:40 +0900
Subject: [PATCH 1219/3199] save

---
 scheduler/src/lib.rs | 18 +++++++++---------
 1 file changed, 9 insertions(+), 9 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 400141cc7a5a18..8708ebe92ab53d 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -438,7 +438,7 @@ impl Task {
         TaskInQueue::new(Self { unique_weight, tx, contention_count: 0, uncontended: Default::default(), sequence_time: std::sync::atomic::AtomicUsize::new(usize::max_value()) })
     }
 
-    pub fn observe_clock(&self, clock: usize) {
+    pub fn record_sequence_time(&self, clock: usize) {
         self.sequence_time.store(clock, std::sync::atomic::Ordering::SeqCst);
     }
 
@@ -890,10 +890,10 @@ impl ScheduleStage {
         weighted_tx: (Weight, TaskInQueue),
         runnable_queue: &mut TaskQueue,
         unique_key: &mut u64,
-        queue_clock: &mut usize,
+        sequence_time: &mut usize,
     ) {
-        weighted_tx.1.observe_clock(*queue_clock);
-        *queue_clock = queue_clock.checked_add(1).unwrap();
+        weighted_tx.1.record_sequence_time(*sequence_time);
+        *sequence_time = sequence_time.checked_add(1).unwrap();
         Self::push_to_runnable_queue(weighted_tx, runnable_queue, unique_key)
     }
 
@@ -909,7 +909,7 @@ impl ScheduleStage {
         let mut executing_queue_count = 0;
         let mut current_unique_key = u64::max_value();
         let mut contended_count = 0;
-        let mut queue_clock = 0;
+        let mut sequence_time = 0;
         let (ee_sender, ee_receiver) = crossbeam_channel::unbounded::<Box<ExecutionEnvironment>>();
 
         let (to_next_stage, maybe_jon_handle) = if let Some(to_next_stage) = maybe_to_next_stage {
@@ -948,14 +948,14 @@ impl ScheduleStage {
                                 to_next_stage.send(processed_execution_environment).unwrap();
                             }
 
-                            Self::register_runnable_task(weighted_tx, runnable_queue, &mut current_unique_key, &mut queue_clock);
+                            Self::register_runnable_task(weighted_tx, runnable_queue, &mut current_unique_key, &mut sequence_time);
 
                             while from.len() > 0 && from_exec.len() == 0 {
                                let i = from.recv().unwrap();
                                 match i {
                                     Multiplexed::FromPrevious(weighted_tx) => {
                                         trace!("recv from previous (after starvation)");
-                                        Self::register_runnable_task(weighted_tx, runnable_queue, &mut current_unique_key, &mut queue_clock);
+                                        Self::register_runnable_task(weighted_tx, runnable_queue, &mut current_unique_key, &mut sequence_time);
                                     }
                                     Multiplexed::FromPreviousBatched(vvv) => {
                                         unreachable!();
@@ -996,7 +996,7 @@ impl ScheduleStage {
                         match i {
                             Multiplexed::FromPrevious(weighted_tx) => {
                                 trace!("recv from previous (after starvation)");
-                                Self::register_runnable_task(weighted_tx, runnable_queue, &mut current_unique_key, &mut queue_clock);
+                                Self::register_runnable_task(weighted_tx, runnable_queue, &mut current_unique_key, &mut sequence_time);
                             }
                             Multiplexed::FromPreviousBatched(vvv) => {
                                 unreachable!();
@@ -1022,7 +1022,7 @@ impl ScheduleStage {
                         match i {
                             Multiplexed::FromPrevious(weighted_tx) => {
                                 trace!("recv from previous (after starvation)");
-                                Self::register_runnable_task(weighted_tx, runnable_queue, &mut current_unique_key, &mut queue_clock);
+                                Self::register_runnable_task(weighted_tx, runnable_queue, &mut current_unique_key, &mut sequence_time);
                             }
                             Multiplexed::FromPreviousBatched(vvv) => {
                                 unreachable!();

From add34685b577bd49fa9881113eb85115b65eeecd Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Wed, 24 Aug 2022 14:16:45 +0900
Subject: [PATCH 1220/3199] save

---
 transaction-scheduler-bench/src/main.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/transaction-scheduler-bench/src/main.rs b/transaction-scheduler-bench/src/main.rs
index 8c6ea01f010db6..bffe74d9b0acda 100644
--- a/transaction-scheduler-bench/src/main.rs
+++ b/transaction-scheduler-bench/src/main.rs
@@ -370,7 +370,7 @@ fn handle_transaction_batch(
     completed_transaction_sender.0
         .send(transaction_batch)
         .unwrap();
-    trace!("send from execute: {:?}", uq);
+    error!("send from execute substage: {:?} seq: {}", uq, transaction_batch.sequence_time());
 }
 
 const NUM_SENDERS: usize = 2;

From cad9362a1f9708315be6d0583a987a0a15dcaad1 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Wed, 24 Aug 2022 14:16:56 +0900
Subject: [PATCH 1221/3199] save

---
 transaction-scheduler-bench/src/main.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/transaction-scheduler-bench/src/main.rs b/transaction-scheduler-bench/src/main.rs
index bffe74d9b0acda..fb48c1118826b4 100644
--- a/transaction-scheduler-bench/src/main.rs
+++ b/transaction-scheduler-bench/src/main.rs
@@ -370,7 +370,7 @@ fn handle_transaction_batch(
     completed_transaction_sender.0
         .send(transaction_batch)
         .unwrap();
-    error!("send from execute substage: {:?} seq: {}", uq, transaction_batch.sequence_time());
+    error!("send from execute substage: {:?} seq: {}", uq, transaction_batch.task.sequence_time());
 }
 
 const NUM_SENDERS: usize = 2;

From 74c58fab2da3b65fd2c41c1dd0c23b35317b3054 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Wed, 24 Aug 2022 14:17:04 +0900
Subject: [PATCH 1222/3199] save

---
 transaction-scheduler-bench/src/main.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/transaction-scheduler-bench/src/main.rs b/transaction-scheduler-bench/src/main.rs
index fb48c1118826b4..1767c1488bfdeb 100644
--- a/transaction-scheduler-bench/src/main.rs
+++ b/transaction-scheduler-bench/src/main.rs
@@ -367,10 +367,10 @@ fn handle_transaction_batch(
             ()
         });
     }
+    error!("send from execute substage: {:?} seq: {}", uq, transaction_batch.task.sequence_time());
     completed_transaction_sender.0
         .send(transaction_batch)
         .unwrap();
-    error!("send from execute substage: {:?} seq: {}", uq, transaction_batch.task.sequence_time());
 }
 
 const NUM_SENDERS: usize = 2;

From 671644f24a968903310226cb64ec00390209830c Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Wed, 24 Aug 2022 14:19:08 +0900
Subject: [PATCH 1223/3199] save

---
 scheduler/src/lib.rs                    | 3 ++-
 transaction-scheduler-bench/src/main.rs | 2 +-
 2 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 8708ebe92ab53d..0db56f903b0860 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -431,7 +431,7 @@ pub struct Task {
 // sequence_time
 // queue_time
 // execute_time
-// terminate_time
+// commit_time
 
 impl Task {
     pub fn new_for_queue(unique_weight: UniqueWeight, tx: Box<(SanitizedTransaction, Vec<LockAttempt>)>) -> std::sync::Arc<Self> {
@@ -861,6 +861,7 @@ impl ScheduleStage {
     #[inline(never)]
     fn commit_result(ee: &mut ExecutionEnvironment, address_book: &mut AddressBook) {
         // do par()-ly?
+        trace!("commit: {}", ee.task.sequence_time())
         Self::unlock_after_execution(address_book, &mut ee.lock_attempts);
         ee.task.mark_as_finished();
         // block-wide qos validation will be done here
diff --git a/transaction-scheduler-bench/src/main.rs b/transaction-scheduler-bench/src/main.rs
index 1767c1488bfdeb..bfae1fc63521d5 100644
--- a/transaction-scheduler-bench/src/main.rs
+++ b/transaction-scheduler-bench/src/main.rs
@@ -367,7 +367,7 @@ fn handle_transaction_batch(
             ()
         });
     }
-    error!("send from execute substage: {:?} seq: {}", uq, transaction_batch.task.sequence_time());
+    //error!("send from execute substage: {:?} seq: {}", uq, transaction_batch.task.sequence_time());
     completed_transaction_sender.0
         .send(transaction_batch)
         .unwrap();

From 48a69e2366a18ecac0c78b57fd886da0db6c1eec Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Wed, 24 Aug 2022 14:19:14 +0900
Subject: [PATCH 1224/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 0db56f903b0860..6782babcec91a9 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -861,7 +861,7 @@ impl ScheduleStage {
     #[inline(never)]
     fn commit_result(ee: &mut ExecutionEnvironment, address_book: &mut AddressBook) {
         // do par()-ly?
-        trace!("commit: {}", ee.task.sequence_time())
+        trace!("commit: {}", ee.task.sequence_time());
         Self::unlock_after_execution(address_book, &mut ee.lock_attempts);
         ee.task.mark_as_finished();
         // block-wide qos validation will be done here

From d3168183748cef007ccae6698a9b7022102a6f3c Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Wed, 24 Aug 2022 14:26:24 +0900
Subject: [PATCH 1225/3199] save

---
 scheduler/src/lib.rs | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 6782babcec91a9..3fea25a19474f1 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -859,9 +859,9 @@ impl ScheduleStage {
     }
 
     #[inline(never)]
-    fn commit_result(ee: &mut ExecutionEnvironment, address_book: &mut AddressBook) {
+    fn commit_completed_execution(ee: &mut ExecutionEnvironment, address_book: &mut AddressBook) {
         // do par()-ly?
-        trace!("commit: {}", ee.task.sequence_time());
+        trace!("commit: seq: {}", ee.task.sequence_time());
         Self::unlock_after_execution(address_book, &mut ee.lock_attempts);
         ee.task.mark_as_finished();
         // block-wide qos validation will be done here
@@ -944,7 +944,7 @@ impl ScheduleStage {
                                 trace!("recv from execute: {:?}", processed_execution_environment.unique_weight);
                                 executing_queue_count -= 1;
 
-                                Self::commit_result(&mut processed_execution_environment, address_book);
+                                Self::commit_completed_execution(&mut processed_execution_environment, address_book);
                                 // async-ly propagate the result to rpc subsystems
                                 to_next_stage.send(processed_execution_environment).unwrap();
                             }
@@ -975,7 +975,7 @@ impl ScheduleStage {
                         trace!("recv from execute: {:?}", processed_execution_environment.unique_weight);
                         executing_queue_count -= 1;
 
-                        Self::commit_result(&mut processed_execution_environment, address_book);
+                        Self::commit_completed_execution(&mut processed_execution_environment, address_book);
                         // async-ly propagate the result to rpc subsystems
                         to_next_stage.send(processed_execution_environment).unwrap();
                         if false && from_exec.len() > 0 {

From b8389c5d3094d98f43d08a4beebc97ccda92b2e0 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Wed, 24 Aug 2022 14:29:14 +0900
Subject: [PATCH 1226/3199] save

---
 scheduler/src/lib.rs | 2 --
 1 file changed, 2 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 3fea25a19474f1..8582c0ebae42ba 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -739,7 +739,6 @@ impl ScheduleStage {
                     address_book,
                     &unique_weight,
                     &mut next_task.tx.1,
-                    provisional_count,
                 );
                 drop(next_task);
                 address_book.provisioning_trackers.insert(unique_weight, (ProvisioningTracker::new(provisional_count), arc_next_task));
@@ -770,7 +769,6 @@ impl ScheduleStage {
         address_book: &mut AddressBook,
         unique_weight: &UniqueWeight,
         lock_attempts: &mut Vec<LockAttempt>,
-        provisional_count: usize,
     ) {
         for mut l in lock_attempts {
             match l.status {

From 5f73e11dc3b849a7ca68d4ca7222e5c058baab16 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Wed, 24 Aug 2022 14:31:42 +0900
Subject: [PATCH 1227/3199] save

---
 scheduler/src/lib.rs | 12 +++++++++++-
 1 file changed, 11 insertions(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 8582c0ebae42ba..014b420a25705f 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -426,6 +426,7 @@ pub struct Task {
     pub contention_count: usize,
     pub uncontended: std::sync::atomic::AtomicUsize,
     pub sequence_time: std::sync::atomic::AtomicUsize,
+    pub execute_time: std::sync::atomic::AtomicUsize,
 }
 
 // sequence_time
@@ -435,7 +436,7 @@ pub struct Task {
 
 impl Task {
     pub fn new_for_queue(unique_weight: UniqueWeight, tx: Box<(SanitizedTransaction, Vec<LockAttempt>)>) -> std::sync::Arc<Self> {
-        TaskInQueue::new(Self { unique_weight, tx, contention_count: 0, uncontended: Default::default(), sequence_time: std::sync::atomic::AtomicUsize::new(usize::max_value()) })
+        TaskInQueue::new(Self { unique_weight, tx, contention_count: 0, uncontended: Default::default(), sequence_time: std::sync::atomic::AtomicUsize::new(usize::max_value()), execute_time: std::sync::atomic::AtomicUsize::new(usize::max_value()) })
     }
 
     pub fn record_sequence_time(&self, clock: usize) {
@@ -446,6 +447,14 @@ impl Task {
         self.sequence_time.load(std::sync::atomic::Ordering::SeqCst)
     }
 
+    pub fn record_execute_time(&self, clock: usize) {
+        self.execute_time.store(clock, std::sync::atomic::Ordering::SeqCst);
+    }
+
+    pub fn execute_time(&self) -> usize {
+        self.execute_time.load(std::sync::atomic::Ordering::SeqCst)
+    }
+
 
     pub fn clone_for_test(&self) -> Self {
         Self {
@@ -454,6 +463,7 @@ impl Task {
             contention_count: Default::default(),
             uncontended: Default::default(),
             sequence_time: std::sync::atomic::AtomicUsize::new(usize::max_value()),
+            execute_time: std::sync::atomic::AtomicUsize::new(usize::max_value()),
         }
     }
 

From 22888acac217a509a9bfa37b62c482a95a802c78 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Wed, 24 Aug 2022 14:33:34 +0900
Subject: [PATCH 1228/3199] save

---
 scheduler/src/lib.rs | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 014b420a25705f..0d2869a535cb31 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -854,9 +854,11 @@ impl ScheduleStage {
         unique_weight: UniqueWeight,
         task: TaskInQueue,
         lock_attempts: Vec<LockAttempt>,
+        execute_clock: &mut usize;
     ) -> Box<ExecutionEnvironment> {
         let mut rng = rand::thread_rng();
         // load account now from AccountsDb
+        task.record_execute_time(*execute_clock);
 
         Box::new(ExecutionEnvironment {
             task,

From f75f49bf01a864ffc85fe1a51c394b1f2af29840 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Wed, 24 Aug 2022 14:34:24 +0900
Subject: [PATCH 1229/3199] save

---
 scheduler/src/lib.rs | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 0d2869a535cb31..9ddcd5d4f23f36 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -889,10 +889,11 @@ impl ScheduleStage {
         address_book: &mut AddressBook,
         contended_count: &mut usize,
         prefer_immediate: bool,
+        execute_clock: &mut usize,
     ) -> Option<Box<ExecutionEnvironment>> {
         let maybe_ee =
             Self::pop_from_queue_then_lock(runnable_queue, address_book, contended_count, prefer_immediate)
-                .map(|(uw, t,ll)| Self::prepare_scheduled_execution(address_book, uw, t, ll));
+                .map(|(uw, t,ll)| Self::prepare_scheduled_execution(address_book, uw, t, ll, execute_clock));
         maybe_ee
     }
 
@@ -921,6 +922,7 @@ impl ScheduleStage {
         let mut current_unique_key = u64::max_value();
         let mut contended_count = 0;
         let mut sequence_time = 0;
+        let mut execute_clock = 0;
         let (ee_sender, ee_receiver) = crossbeam_channel::unbounded::<Box<ExecutionEnvironment>>();
 
         let (to_next_stage, maybe_jon_handle) = if let Some(to_next_stage) = maybe_to_next_stage {
@@ -1019,7 +1021,7 @@ impl ScheduleStage {
 
                 let prefer_immediate = address_book.provisioning_trackers.len()/4 > executing_queue_count;
                 let maybe_ee =
-                    Self::schedule_next_execution(runnable_queue, address_book, &mut contended_count, prefer_immediate);
+                    Self::schedule_next_execution(runnable_queue, address_book, &mut contended_count, prefer_immediate, &mut execute_clock);
 
                 if let Some(ee) = maybe_ee {
                     trace!("send to execute");

From a6f88a7f4c5cda20a2529c259bc3198eee4d1e72 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Wed, 24 Aug 2022 14:35:49 +0900
Subject: [PATCH 1230/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 9ddcd5d4f23f36..1809eb16f9febc 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -854,7 +854,7 @@ impl ScheduleStage {
         unique_weight: UniqueWeight,
         task: TaskInQueue,
         lock_attempts: Vec<LockAttempt>,
-        execute_clock: &mut usize;
+        execute_clock: &mut usize,
     ) -> Box<ExecutionEnvironment> {
         let mut rng = rand::thread_rng();
         // load account now from AccountsDb

From 51f25c0b968a979d47826973b618b76d6d631ffd Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Wed, 24 Aug 2022 14:37:00 +0900
Subject: [PATCH 1231/3199] save

---
 scheduler/src/lib.rs | 1 +
 1 file changed, 1 insertion(+)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 1809eb16f9febc..d59ae435d1de92 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -859,6 +859,7 @@ impl ScheduleStage {
         let mut rng = rand::thread_rng();
         // load account now from AccountsDb
         task.record_execute_time(*execute_clock);
+        *execute_clock = execute_clock.checked_add(1).unwrap();
 
         Box::new(ExecutionEnvironment {
             task,

From 26d516d862ad328539447cbd8956f227c5940edc Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Wed, 24 Aug 2022 14:38:27 +0900
Subject: [PATCH 1232/3199] save

---
 scheduler/src/lib.rs | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index d59ae435d1de92..144c6c9a31f6c0 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -870,9 +870,9 @@ impl ScheduleStage {
     }
 
     #[inline(never)]
-    fn commit_completed_execution(ee: &mut ExecutionEnvironment, address_book: &mut AddressBook) {
+    fn commit_completed_execution(ee: &mut ExecutionEnvironment, address_book: &mut AddressBook, current_exec_clock: &usize) {
         // do par()-ly?
-        trace!("commit: seq: {}", ee.task.sequence_time());
+        trace!("commit: seq: {}, exec: {}->{}", ee.task.sequence_time(), ee.task.execute_time(), current_exec_clock);
         Self::unlock_after_execution(address_book, &mut ee.lock_attempts);
         ee.task.mark_as_finished();
         // block-wide qos validation will be done here
@@ -957,7 +957,7 @@ impl ScheduleStage {
                                 trace!("recv from execute: {:?}", processed_execution_environment.unique_weight);
                                 executing_queue_count -= 1;
 
-                                Self::commit_completed_execution(&mut processed_execution_environment, address_book);
+                                Self::commit_completed_execution(&mut processed_execution_environment, address_book, &execute_clock);
                                 // async-ly propagate the result to rpc subsystems
                                 to_next_stage.send(processed_execution_environment).unwrap();
                             }
@@ -988,7 +988,7 @@ impl ScheduleStage {
                         trace!("recv from execute: {:?}", processed_execution_environment.unique_weight);
                         executing_queue_count -= 1;
 
-                        Self::commit_completed_execution(&mut processed_execution_environment, address_book);
+                        Self::commit_completed_execution(&mut processed_execution_environment, address_book, &execute_clock);
                         // async-ly propagate the result to rpc subsystems
                         to_next_stage.send(processed_execution_environment).unwrap();
                         if false && from_exec.len() > 0 {

From 5268f9b41b73b63f503f05aa7b13e2de701812c3 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Wed, 24 Aug 2022 14:38:34 +0900
Subject: [PATCH 1233/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 144c6c9a31f6c0..ba257ebe03a2ab 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -872,7 +872,7 @@ impl ScheduleStage {
     #[inline(never)]
     fn commit_completed_execution(ee: &mut ExecutionEnvironment, address_book: &mut AddressBook, current_exec_clock: &usize) {
         // do par()-ly?
-        trace!("commit: seq: {}, exec: {}->{}", ee.task.sequence_time(), ee.task.execute_time(), current_exec_clock);
+        info!("commit: seq: {}, exec: {}->{}", ee.task.sequence_time(), ee.task.execute_time(), current_exec_clock);
         Self::unlock_after_execution(address_book, &mut ee.lock_attempts);
         ee.task.mark_as_finished();
         // block-wide qos validation will be done here

From e48ae26bd0e02d613c5bdb5ec60373c45c5caab2 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Wed, 24 Aug 2022 14:39:47 +0900
Subject: [PATCH 1234/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index ba257ebe03a2ab..ab2d128d5acd7c 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -872,7 +872,7 @@ impl ScheduleStage {
     #[inline(never)]
     fn commit_completed_execution(ee: &mut ExecutionEnvironment, address_book: &mut AddressBook, current_exec_clock: &usize) {
         // do par()-ly?
-        info!("commit: seq: {}, exec: {}->{}", ee.task.sequence_time(), ee.task.execute_time(), current_exec_clock);
+        info!("commit: seq: {}, exec: [{}..{}; {}]", ee.task.sequence_time(), ee.task.execute_time(), current_exec_clock, current_exec_clock - ee.task.execute_time());
         Self::unlock_after_execution(address_book, &mut ee.lock_attempts);
         ee.task.mark_as_finished();
         // block-wide qos validation will be done here

From a2543e35056b54dbc13bd98cc82e691726bb2708 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Wed, 24 Aug 2022 14:43:44 +0900
Subject: [PATCH 1235/3199] save

---
 scheduler/src/lib.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index ab2d128d5acd7c..73dba4178868b6 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -870,9 +870,9 @@ impl ScheduleStage {
     }
 
     #[inline(never)]
-    fn commit_completed_execution(ee: &mut ExecutionEnvironment, address_book: &mut AddressBook, current_exec_clock: &usize) {
+    fn commit_completed_execution(ee: &mut ExecutionEnvironment, address_book: &mut AddressBook, commit_time: &usize) {
         // do par()-ly?
-        info!("commit: seq: {}, exec: [{}..{}; {}]", ee.task.sequence_time(), ee.task.execute_time(), current_exec_clock, current_exec_clock - ee.task.execute_time());
+        info!("commit: seq: {}, exec: [{}..{}; {}]", ee.task.sequence_time(), ee.task.execute_time(), commit_time, commit_time - ee.task.execute_time());
         Self::unlock_after_execution(address_book, &mut ee.lock_attempts);
         ee.task.mark_as_finished();
         // block-wide qos validation will be done here

From a0608f2aa772afbf3c214be2359e687c740f5dac Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Wed, 24 Aug 2022 14:47:06 +0900
Subject: [PATCH 1236/3199] save

---
 scheduler/src/lib.rs | 13 ++++++++++++-
 1 file changed, 12 insertions(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 73dba4178868b6..6dfe8858b42ef4 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -426,6 +426,7 @@ pub struct Task {
     pub contention_count: usize,
     pub uncontended: std::sync::atomic::AtomicUsize,
     pub sequence_time: std::sync::atomic::AtomicUsize,
+    pub queue_time: std::sync::atomic::AtomicUsize,
     pub execute_time: std::sync::atomic::AtomicUsize,
 }
 
@@ -436,7 +437,7 @@ pub struct Task {
 
 impl Task {
     pub fn new_for_queue(unique_weight: UniqueWeight, tx: Box<(SanitizedTransaction, Vec<LockAttempt>)>) -> std::sync::Arc<Self> {
-        TaskInQueue::new(Self { unique_weight, tx, contention_count: 0, uncontended: Default::default(), sequence_time: std::sync::atomic::AtomicUsize::new(usize::max_value()), execute_time: std::sync::atomic::AtomicUsize::new(usize::max_value()) })
+        TaskInQueue::new(Self { unique_weight, tx, contention_count: 0, uncontended: Default::default(), sequence_time: std::sync::atomic::AtomicUsize::new(usize::max_value()), queue_time: std::sync::atomic::AtomicUsize::new(usize::max_value()), execute_time: std::sync::atomic::AtomicUsize::new(usize::max_value()) })
     }
 
     pub fn record_sequence_time(&self, clock: usize) {
@@ -447,6 +448,15 @@ impl Task {
         self.sequence_time.load(std::sync::atomic::Ordering::SeqCst)
     }
 
+    pub fn record_queue_time(&self, clock: usize) {
+        self.queue_time.store(clock, std::sync::atomic::Ordering::SeqCst);
+    }
+
+    pub fn queue_time(&self) -> usize {
+        self.queue_time.load(std::sync::atomic::Ordering::SeqCst)
+    }
+
+
     pub fn record_execute_time(&self, clock: usize) {
         self.execute_time.store(clock, std::sync::atomic::Ordering::SeqCst);
     }
@@ -463,6 +473,7 @@ impl Task {
             contention_count: Default::default(),
             uncontended: Default::default(),
             sequence_time: std::sync::atomic::AtomicUsize::new(usize::max_value()),
+            queue_time: std::sync::atomic::AtomicUsize::new(usize::max_value()),
             execute_time: std::sync::atomic::AtomicUsize::new(usize::max_value()),
         }
     }

From 7827a3bdd9d6b5f7d7a7abeac506514ced3e277b Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Wed, 24 Aug 2022 14:49:13 +0900
Subject: [PATCH 1237/3199] save

---
 scheduler/src/lib.rs | 12 +++++++-----
 1 file changed, 7 insertions(+), 5 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 6dfe8858b42ef4..bd30bb96099d7b 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -430,10 +430,10 @@ pub struct Task {
     pub execute_time: std::sync::atomic::AtomicUsize,
 }
 
-// sequence_time
-// queue_time
-// execute_time
-// commit_time
+// sequence_time -> seq clock
+// queue_time -> queue clock
+// execute_time ---> exec clock
+// commit_time  -+
 
 impl Task {
     pub fn new_for_queue(unique_weight: UniqueWeight, tx: Box<(SanitizedTransaction, Vec<LockAttempt>)>) -> std::sync::Arc<Self> {
@@ -456,7 +456,6 @@ impl Task {
         self.queue_time.load(std::sync::atomic::Ordering::SeqCst)
     }
 
-
     pub fn record_execute_time(&self, clock: usize) {
         self.execute_time.store(clock, std::sync::atomic::Ordering::SeqCst);
     }
@@ -884,8 +883,11 @@ impl ScheduleStage {
     fn commit_completed_execution(ee: &mut ExecutionEnvironment, address_book: &mut AddressBook, commit_time: &usize) {
         // do par()-ly?
         info!("commit: seq: {}, exec: [{}..{}; {}]", ee.task.sequence_time(), ee.task.execute_time(), commit_time, commit_time - ee.task.execute_time());
+
+        // which order for data race free?: unlocking / marking
         Self::unlock_after_execution(address_book, &mut ee.lock_attempts);
         ee.task.mark_as_finished();
+
         // block-wide qos validation will be done here
         // if error risen..:
         //   don't commit the tx for banking and potentially finish scheduling at block max cu

From fb2070f1dfdca8ec4e065abd705e104f33ff367d Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Wed, 24 Aug 2022 14:54:40 +0900
Subject: [PATCH 1238/3199] save

---
 scheduler/src/lib.rs | 11 +++++++++--
 1 file changed, 9 insertions(+), 2 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index bd30bb96099d7b..b89fc250d6f12f 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -693,6 +693,7 @@ impl ScheduleStage {
         address_book: &mut AddressBook,
         contended_count: &mut usize,
         prefer_immediate: bool,
+        queue_clock: &mut usize,
     ) -> Option<(UniqueWeight, TaskInQueue, Vec<LockAttempt>)> {
         if let Some(mut a) = address_book.fulfilled_provisional_task_ids.pop_last() {
             trace!("expediate pop from provisional queue [rest: {}]", address_book.fulfilled_provisional_task_ids.len());
@@ -709,6 +710,10 @@ impl ScheduleStage {
         loop {
         if let Some((from_runnable, mut arc_next_task)) = Self::select_next_task(runnable_queue, address_book) {
             trace!("pop loop iteration");
+            if from_runnable {
+                arc_next_task.record_queue_time(queue_clock);
+                *queue_clock = queue_clock.checked_add(1).unwrap();
+            }
             let next_task = unsafe { TaskInQueue::get_mut_unchecked(&mut arc_next_task) };
             let unique_weight = next_task.unique_weight;
             let message_hash = next_task.tx.0.message_hash();
@@ -903,10 +908,11 @@ impl ScheduleStage {
         address_book: &mut AddressBook,
         contended_count: &mut usize,
         prefer_immediate: bool,
+        queue_clock: &mut usize,
         execute_clock: &mut usize,
     ) -> Option<Box<ExecutionEnvironment>> {
         let maybe_ee =
-            Self::pop_from_queue_then_lock(runnable_queue, address_book, contended_count, prefer_immediate)
+            Self::pop_from_queue_then_lock(runnable_queue, address_book, contended_count, prefer_immediate, queue_clock)
                 .map(|(uw, t,ll)| Self::prepare_scheduled_execution(address_book, uw, t, ll, execute_clock));
         maybe_ee
     }
@@ -936,6 +942,7 @@ impl ScheduleStage {
         let mut current_unique_key = u64::max_value();
         let mut contended_count = 0;
         let mut sequence_time = 0;
+        let mut queue_clock = 0;
         let mut execute_clock = 0;
         let (ee_sender, ee_receiver) = crossbeam_channel::unbounded::<Box<ExecutionEnvironment>>();
 
@@ -1035,7 +1042,7 @@ impl ScheduleStage {
 
                 let prefer_immediate = address_book.provisioning_trackers.len()/4 > executing_queue_count;
                 let maybe_ee =
-                    Self::schedule_next_execution(runnable_queue, address_book, &mut contended_count, prefer_immediate, &mut execute_clock);
+                    Self::schedule_next_execution(runnable_queue, address_book, &mut contended_count, prefer_immediate, &mut queue_clock, &mut execute_clock);
 
                 if let Some(ee) = maybe_ee {
                     trace!("send to execute");

From ed8a2cbd2e63b14e7661167ff95a73e014126f33 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Wed, 24 Aug 2022 14:54:52 +0900
Subject: [PATCH 1239/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index b89fc250d6f12f..fac06fa53de738 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -711,7 +711,7 @@ impl ScheduleStage {
         if let Some((from_runnable, mut arc_next_task)) = Self::select_next_task(runnable_queue, address_book) {
             trace!("pop loop iteration");
             if from_runnable {
-                arc_next_task.record_queue_time(queue_clock);
+                arc_next_task.record_queue_time(*queue_clock);
                 *queue_clock = queue_clock.checked_add(1).unwrap();
             }
             let next_task = unsafe { TaskInQueue::get_mut_unchecked(&mut arc_next_task) };

From 3a90a3771fa6eb265f36d75b94260eebc67a1dc3 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Wed, 24 Aug 2022 14:59:43 +0900
Subject: [PATCH 1240/3199] save

---
 scheduler/src/lib.rs | 14 +++++++++-----
 1 file changed, 9 insertions(+), 5 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index fac06fa53de738..15592f92a5b68e 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -427,6 +427,7 @@ pub struct Task {
     pub uncontended: std::sync::atomic::AtomicUsize,
     pub sequence_time: std::sync::atomic::AtomicUsize,
     pub queue_time: std::sync::atomic::AtomicUsize,
+    pub queue_end_time: std::sync::atomic::AtomicUsize,
     pub execute_time: std::sync::atomic::AtomicUsize,
 }
 
@@ -437,7 +438,7 @@ pub struct Task {
 
 impl Task {
     pub fn new_for_queue(unique_weight: UniqueWeight, tx: Box<(SanitizedTransaction, Vec<LockAttempt>)>) -> std::sync::Arc<Self> {
-        TaskInQueue::new(Self { unique_weight, tx, contention_count: 0, uncontended: Default::default(), sequence_time: std::sync::atomic::AtomicUsize::new(usize::max_value()), queue_time: std::sync::atomic::AtomicUsize::new(usize::max_value()), execute_time: std::sync::atomic::AtomicUsize::new(usize::max_value()) })
+        TaskInQueue::new(Self { unique_weight, tx, contention_count: 0, uncontended: Default::default(), sequence_time: std::sync::atomic::AtomicUsize::new(usize::max_value()), queue_time: std::sync::atomic::AtomicUsize::new(usize::max_value()), queue_end_time: std::sync::atomic::AtomicUsize::new(usize::max_value()), execute_time: std::sync::atomic::AtomicUsize::new(usize::max_value()) })
     }
 
     pub fn record_sequence_time(&self, clock: usize) {
@@ -456,8 +457,9 @@ impl Task {
         self.queue_time.load(std::sync::atomic::Ordering::SeqCst)
     }
 
-    pub fn record_execute_time(&self, clock: usize) {
-        self.execute_time.store(clock, std::sync::atomic::Ordering::SeqCst);
+    pub fn record_execute_time(&self, queue_clock: usize, execute_clock: usize) {
+        self.queue_end_time.load(queue_clock, std::sync::atomic::Ordering::SeqCst)
+        self.execute_time.store(execute_clock, std::sync::atomic::Ordering::SeqCst);
     }
 
     pub fn execute_time(&self) -> usize {
@@ -473,6 +475,7 @@ impl Task {
             uncontended: Default::default(),
             sequence_time: std::sync::atomic::AtomicUsize::new(usize::max_value()),
             queue_time: std::sync::atomic::AtomicUsize::new(usize::max_value()),
+            queue_end_time: std::sync::atomic::AtomicUsize::new(usize::max_value()),
             execute_time: std::sync::atomic::AtomicUsize::new(usize::max_value()),
         }
     }
@@ -869,11 +872,12 @@ impl ScheduleStage {
         unique_weight: UniqueWeight,
         task: TaskInQueue,
         lock_attempts: Vec<LockAttempt>,
+        queue_clock: &usize,
         execute_clock: &mut usize,
     ) -> Box<ExecutionEnvironment> {
         let mut rng = rand::thread_rng();
         // load account now from AccountsDb
-        task.record_execute_time(*execute_clock);
+        task.record_execute_time(queue_clock, *execute_clock);
         *execute_clock = execute_clock.checked_add(1).unwrap();
 
         Box::new(ExecutionEnvironment {
@@ -913,7 +917,7 @@ impl ScheduleStage {
     ) -> Option<Box<ExecutionEnvironment>> {
         let maybe_ee =
             Self::pop_from_queue_then_lock(runnable_queue, address_book, contended_count, prefer_immediate, queue_clock)
-                .map(|(uw, t,ll)| Self::prepare_scheduled_execution(address_book, uw, t, ll, execute_clock));
+                .map(|(uw, t,ll)| Self::prepare_scheduled_execution(address_book, uw, t, ll, queue_clock, execute_clock));
         maybe_ee
     }
 

From 8aa760db02b845fb44c046815eff46f5fc572f44 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Wed, 24 Aug 2022 15:00:04 +0900
Subject: [PATCH 1241/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 15592f92a5b68e..cc17de6fa08056 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -458,7 +458,7 @@ impl Task {
     }
 
     pub fn record_execute_time(&self, queue_clock: usize, execute_clock: usize) {
-        self.queue_end_time.load(queue_clock, std::sync::atomic::Ordering::SeqCst)
+        self.queue_end_time.load(queue_clock, std::sync::atomic::Ordering::SeqCst);
         self.execute_time.store(execute_clock, std::sync::atomic::Ordering::SeqCst);
     }
 

From 3c9932cbc474310d39bb30baa4b69aa60c0f9910 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Wed, 24 Aug 2022 15:00:23 +0900
Subject: [PATCH 1242/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index cc17de6fa08056..ac7eeeb6c4ea5c 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -877,7 +877,7 @@ impl ScheduleStage {
     ) -> Box<ExecutionEnvironment> {
         let mut rng = rand::thread_rng();
         // load account now from AccountsDb
-        task.record_execute_time(queue_clock, *execute_clock);
+        task.record_execute_time(*queue_clock, *execute_clock);
         *execute_clock = execute_clock.checked_add(1).unwrap();
 
         Box::new(ExecutionEnvironment {

From 802edf9d19bad1d1851ebbf468510703c861fb87 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Wed, 24 Aug 2022 15:00:37 +0900
Subject: [PATCH 1243/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index ac7eeeb6c4ea5c..d1f3253fad85c0 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -458,7 +458,7 @@ impl Task {
     }
 
     pub fn record_execute_time(&self, queue_clock: usize, execute_clock: usize) {
-        self.queue_end_time.load(queue_clock, std::sync::atomic::Ordering::SeqCst);
+        self.queue_end_time.store(queue_clock, std::sync::atomic::Ordering::SeqCst);
         self.execute_time.store(execute_clock, std::sync::atomic::Ordering::SeqCst);
     }
 

From d37c107d247fc35b16625a31d6132363e15d15bf Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Wed, 24 Aug 2022 15:02:06 +0900
Subject: [PATCH 1244/3199] save

---
 scheduler/src/lib.rs | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index d1f3253fad85c0..66aaee726ad96c 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -891,7 +891,10 @@ impl ScheduleStage {
     #[inline(never)]
     fn commit_completed_execution(ee: &mut ExecutionEnvironment, address_book: &mut AddressBook, commit_time: &usize) {
         // do par()-ly?
-        info!("commit: seq: {}, exec: [{}..{}; {}]", ee.task.sequence_time(), ee.task.execute_time(), commit_time, commit_time - ee.task.execute_time());
+        info!("commit: seq: {}, queue: [{}..{}; {}] exec: [{}..{}; {}]", 
+              ee.task.sequence_time(),
+              ee.task.queue_time(), ee.task.queue_end_time(), ee.task.queue_end_time() - ee.task.queue_time(), 
+              ee.task.execute_time(), commit_time, commit_time - ee.task.execute_time());
 
         // which order for data race free?: unlocking / marking
         Self::unlock_after_execution(address_book, &mut ee.lock_attempts);

From 6ec47866e1d5213cb47a8629195010dcfd9a3a3f Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Wed, 24 Aug 2022 15:02:27 +0900
Subject: [PATCH 1245/3199] save

---
 scheduler/src/lib.rs | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 66aaee726ad96c..0df422ecc1bcfc 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -457,6 +457,10 @@ impl Task {
         self.queue_time.load(std::sync::atomic::Ordering::SeqCst)
     }
 
+    pub fn queue_time(&self) -> usize {
+        self.queue_end_time.load(std::sync::atomic::Ordering::SeqCst)
+    }
+
     pub fn record_execute_time(&self, queue_clock: usize, execute_clock: usize) {
         self.queue_end_time.store(queue_clock, std::sync::atomic::Ordering::SeqCst);
         self.execute_time.store(execute_clock, std::sync::atomic::Ordering::SeqCst);

From d80fbc5dc97c43fd1cb8c9ea1bef0a140963a00b Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Wed, 24 Aug 2022 15:02:34 +0900
Subject: [PATCH 1246/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 0df422ecc1bcfc..a9942ead7709d1 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -457,7 +457,7 @@ impl Task {
         self.queue_time.load(std::sync::atomic::Ordering::SeqCst)
     }
 
-    pub fn queue_time(&self) -> usize {
+    pub fn queue_end_time(&self) -> usize {
         self.queue_end_time.load(std::sync::atomic::Ordering::SeqCst)
     }
 

From 4c8a6312ff62b78ffe985cc2b384a5661e4b803e Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Wed, 24 Aug 2022 15:05:03 +0900
Subject: [PATCH 1247/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index a9942ead7709d1..d2a79e3fb190b4 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -895,7 +895,7 @@ impl ScheduleStage {
     #[inline(never)]
     fn commit_completed_execution(ee: &mut ExecutionEnvironment, address_book: &mut AddressBook, commit_time: &usize) {
         // do par()-ly?
-        info!("commit: seq: {}, queue: [{}..{}; {}] exec: [{}..{}; {}]", 
+        info!("commit: seq: {}, queue: [{}qt..{}qt; {}qd] exec: [{}et..{}et; {}ed]", 
               ee.task.sequence_time(),
               ee.task.queue_time(), ee.task.queue_end_time(), ee.task.queue_end_time() - ee.task.queue_time(), 
               ee.task.execute_time(), commit_time, commit_time - ee.task.execute_time());

From 9b7de73478c72e93cc6eb11a352bed4b597fe224 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Wed, 24 Aug 2022 15:08:40 +0900
Subject: [PATCH 1248/3199] save

---
 scheduler/src/lib.rs | 15 ++++++++++-----
 1 file changed, 10 insertions(+), 5 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index d2a79e3fb190b4..7c4646a5e40e95 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -426,6 +426,7 @@ pub struct Task {
     pub contention_count: usize,
     pub uncontended: std::sync::atomic::AtomicUsize,
     pub sequence_time: std::sync::atomic::AtomicUsize,
+    pub sequence_end_time: std::sync::atomic::AtomicUsize,
     pub queue_time: std::sync::atomic::AtomicUsize,
     pub queue_end_time: std::sync::atomic::AtomicUsize,
     pub execute_time: std::sync::atomic::AtomicUsize,
@@ -438,7 +439,7 @@ pub struct Task {
 
 impl Task {
     pub fn new_for_queue(unique_weight: UniqueWeight, tx: Box<(SanitizedTransaction, Vec<LockAttempt>)>) -> std::sync::Arc<Self> {
-        TaskInQueue::new(Self { unique_weight, tx, contention_count: 0, uncontended: Default::default(), sequence_time: std::sync::atomic::AtomicUsize::new(usize::max_value()), queue_time: std::sync::atomic::AtomicUsize::new(usize::max_value()), queue_end_time: std::sync::atomic::AtomicUsize::new(usize::max_value()), execute_time: std::sync::atomic::AtomicUsize::new(usize::max_value()) })
+        TaskInQueue::new(Self { unique_weight, tx, contention_count: 0, uncontended: Default::default(), sequence_time: std::sync::atomic::AtomicUsize::new(usize::max_value()), sequence_end_time: std::sync::atomic::AtomicUsize::new(usize::max_value()), queue_time: std::sync::atomic::AtomicUsize::new(usize::max_value()), queue_end_time: std::sync::atomic::AtomicUsize::new(usize::max_value()), execute_time: std::sync::atomic::AtomicUsize::new(usize::max_value()) })
     }
 
     pub fn record_sequence_time(&self, clock: usize) {
@@ -449,8 +450,9 @@ impl Task {
         self.sequence_time.load(std::sync::atomic::Ordering::SeqCst)
     }
 
-    pub fn record_queue_time(&self, clock: usize) {
-        self.queue_time.store(clock, std::sync::atomic::Ordering::SeqCst);
+    pub fn record_queue_time(&self, seq_clock, usize, queue_clock: usize) {
+        self.sequence_end_time.store(seq_clock, std::sync::atomic::Ordering::SeqCst);
+        self.queue_time.store(queue_clock, std::sync::atomic::Ordering::SeqCst);
     }
 
     pub fn queue_time(&self) -> usize {
@@ -478,6 +480,7 @@ impl Task {
             contention_count: Default::default(),
             uncontended: Default::default(),
             sequence_time: std::sync::atomic::AtomicUsize::new(usize::max_value()),
+            sequence_end_time: std::sync::atomic::AtomicUsize::new(usize::max_value()),
             queue_time: std::sync::atomic::AtomicUsize::new(usize::max_value()),
             queue_end_time: std::sync::atomic::AtomicUsize::new(usize::max_value()),
             execute_time: std::sync::atomic::AtomicUsize::new(usize::max_value()),
@@ -700,6 +703,7 @@ impl ScheduleStage {
         address_book: &mut AddressBook,
         contended_count: &mut usize,
         prefer_immediate: bool,
+        sequence_clock: &usize,
         queue_clock: &mut usize,
     ) -> Option<(UniqueWeight, TaskInQueue, Vec<LockAttempt>)> {
         if let Some(mut a) = address_book.fulfilled_provisional_task_ids.pop_last() {
@@ -919,11 +923,12 @@ impl ScheduleStage {
         address_book: &mut AddressBook,
         contended_count: &mut usize,
         prefer_immediate: bool,
+        sequence_time: &usize,
         queue_clock: &mut usize,
         execute_clock: &mut usize,
     ) -> Option<Box<ExecutionEnvironment>> {
         let maybe_ee =
-            Self::pop_from_queue_then_lock(runnable_queue, address_book, contended_count, prefer_immediate, queue_clock)
+            Self::pop_from_queue_then_lock(runnable_queue, address_book, contended_count, prefer_immediate, sequence_time, queue_clock)
                 .map(|(uw, t,ll)| Self::prepare_scheduled_execution(address_book, uw, t, ll, queue_clock, execute_clock));
         maybe_ee
     }
@@ -1053,7 +1058,7 @@ impl ScheduleStage {
 
                 let prefer_immediate = address_book.provisioning_trackers.len()/4 > executing_queue_count;
                 let maybe_ee =
-                    Self::schedule_next_execution(runnable_queue, address_book, &mut contended_count, prefer_immediate, &mut queue_clock, &mut execute_clock);
+                    Self::schedule_next_execution(runnable_queue, address_book, &mut contended_count, prefer_immediate, &sequence_time, &mut queue_clock, &mut execute_clock);
 
                 if let Some(ee) = maybe_ee {
                     trace!("send to execute");

From fe8579a6f9cf62dfc32f2ebe047b8acb3b2d3be8 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Wed, 24 Aug 2022 15:08:53 +0900
Subject: [PATCH 1249/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 7c4646a5e40e95..4d09df1a5392e5 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -450,7 +450,7 @@ impl Task {
         self.sequence_time.load(std::sync::atomic::Ordering::SeqCst)
     }
 
-    pub fn record_queue_time(&self, seq_clock, usize, queue_clock: usize) {
+    pub fn record_queue_time(&self, seq_clock: usize, queue_clock: usize) {
         self.sequence_end_time.store(seq_clock, std::sync::atomic::Ordering::SeqCst);
         self.queue_time.store(queue_clock, std::sync::atomic::Ordering::SeqCst);
     }

From 3d78cf8d682c7c8279dc6bf2cae5702bdf8c69ff Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Wed, 24 Aug 2022 15:09:06 +0900
Subject: [PATCH 1250/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 4d09df1a5392e5..b00619d4413a7d 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -722,7 +722,7 @@ impl ScheduleStage {
         if let Some((from_runnable, mut arc_next_task)) = Self::select_next_task(runnable_queue, address_book) {
             trace!("pop loop iteration");
             if from_runnable {
-                arc_next_task.record_queue_time(*queue_clock);
+                arc_next_task.record_queue_time(*sequence_time, *queue_clock);
                 *queue_clock = queue_clock.checked_add(1).unwrap();
             }
             let next_task = unsafe { TaskInQueue::get_mut_unchecked(&mut arc_next_task) };

From f37b95b5548422d9f2a96ebbe577f397eb8ab3a3 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Wed, 24 Aug 2022 15:09:15 +0900
Subject: [PATCH 1251/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index b00619d4413a7d..a2f03508a90864 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -722,7 +722,7 @@ impl ScheduleStage {
         if let Some((from_runnable, mut arc_next_task)) = Self::select_next_task(runnable_queue, address_book) {
             trace!("pop loop iteration");
             if from_runnable {
-                arc_next_task.record_queue_time(*sequence_time, *queue_clock);
+                arc_next_task.record_queue_time(*sequence_clock, *queue_clock);
                 *queue_clock = queue_clock.checked_add(1).unwrap();
             }
             let next_task = unsafe { TaskInQueue::get_mut_unchecked(&mut arc_next_task) };

From 5de7814b072534a0d4c376e4cbfd0acfeb8d5b05 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Wed, 24 Aug 2022 15:10:26 +0900
Subject: [PATCH 1252/3199] save

---
 scheduler/src/lib.rs | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index a2f03508a90864..adf74ebb9833b2 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -450,6 +450,10 @@ impl Task {
         self.sequence_time.load(std::sync::atomic::Ordering::SeqCst)
     }
 
+    pub fn sequence_end_time(&self) -> usize {
+        self.sequence_end_time.load(std::sync::atomic::Ordering::SeqCst)
+    }
+
     pub fn record_queue_time(&self, seq_clock: usize, queue_clock: usize) {
         self.sequence_end_time.store(seq_clock, std::sync::atomic::Ordering::SeqCst);
         self.queue_time.store(queue_clock, std::sync::atomic::Ordering::SeqCst);
@@ -899,8 +903,10 @@ impl ScheduleStage {
     #[inline(never)]
     fn commit_completed_execution(ee: &mut ExecutionEnvironment, address_book: &mut AddressBook, commit_time: &usize) {
         // do par()-ly?
-        info!("commit: seq: {}, queue: [{}qt..{}qt; {}qd] exec: [{}et..{}et; {}ed]", 
+        info!("commit: seq: [{}st..{}.st; {}sd], queue: [{}qt..{}qt; {}qd] exec: [{}et..{}et; {}ed]", 
               ee.task.sequence_time(),
+              ee.task.sequence_end_time(),
+              ee.task.sequence_end_time() - ee.task.sequence_time(),
               ee.task.queue_time(), ee.task.queue_end_time(), ee.task.queue_end_time() - ee.task.queue_time(), 
               ee.task.execute_time(), commit_time, commit_time - ee.task.execute_time());
 

From 1c32eb9e24697d42ac49eab023491bde751774e4 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Wed, 24 Aug 2022 15:13:25 +0900
Subject: [PATCH 1253/3199] save

---
 scheduler/src/lib.rs | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index adf74ebb9833b2..7d0b3e37e4ab2e 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -901,8 +901,10 @@ impl ScheduleStage {
     }
 
     #[inline(never)]
-    fn commit_completed_execution(ee: &mut ExecutionEnvironment, address_book: &mut AddressBook, commit_time: &usize) {
+    fn commit_completed_execution(ee: &mut ExecutionEnvironment, address_book: &mut AddressBook, commit_time: &mut usize) {
         // do par()-ly?
+        *commit_time = commit_time.checked_add(1).unwrap();
+
         info!("commit: seq: [{}st..{}.st; {}sd], queue: [{}qt..{}qt; {}qd] exec: [{}et..{}et; {}ed]", 
               ee.task.sequence_time(),
               ee.task.sequence_end_time(),

From fb93dfead5044f4e9b34c876b24bdd2a368d033e Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Wed, 24 Aug 2022 15:13:57 +0900
Subject: [PATCH 1254/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 7d0b3e37e4ab2e..90605a80352f00 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -903,7 +903,6 @@ impl ScheduleStage {
     #[inline(never)]
     fn commit_completed_execution(ee: &mut ExecutionEnvironment, address_book: &mut AddressBook, commit_time: &mut usize) {
         // do par()-ly?
-        *commit_time = commit_time.checked_add(1).unwrap();
 
         info!("commit: seq: [{}st..{}.st; {}sd], queue: [{}qt..{}qt; {}qd] exec: [{}et..{}et; {}ed]", 
               ee.task.sequence_time(),
@@ -911,6 +910,7 @@ impl ScheduleStage {
               ee.task.sequence_end_time() - ee.task.sequence_time(),
               ee.task.queue_time(), ee.task.queue_end_time(), ee.task.queue_end_time() - ee.task.queue_time(), 
               ee.task.execute_time(), commit_time, commit_time - ee.task.execute_time());
+        *commit_time = commit_time.checked_add(1).unwrap();
 
         // which order for data race free?: unlocking / marking
         Self::unlock_after_execution(address_book, &mut ee.lock_attempts);

From 605b1d6b1f1494e94371b25381491902f4dde837 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Wed, 24 Aug 2022 15:14:06 +0900
Subject: [PATCH 1255/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 90605a80352f00..7fee45f7d8fe9e 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -909,7 +909,7 @@ impl ScheduleStage {
               ee.task.sequence_end_time(),
               ee.task.sequence_end_time() - ee.task.sequence_time(),
               ee.task.queue_time(), ee.task.queue_end_time(), ee.task.queue_end_time() - ee.task.queue_time(), 
-              ee.task.execute_time(), commit_time, commit_time - ee.task.execute_time());
+              ee.task.execute_time(), *commit_time, *commit_time - ee.task.execute_time());
         *commit_time = commit_time.checked_add(1).unwrap();
 
         // which order for data race free?: unlocking / marking

From bd9212e13f6b10bf689f3d23ca854a18730a64cc Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Wed, 24 Aug 2022 15:14:28 +0900
Subject: [PATCH 1256/3199] save

---
 scheduler/src/lib.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 7fee45f7d8fe9e..dcd68a3d859afb 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1001,7 +1001,7 @@ impl ScheduleStage {
                                 trace!("recv from execute: {:?}", processed_execution_environment.unique_weight);
                                 executing_queue_count -= 1;
 
-                                Self::commit_completed_execution(&mut processed_execution_environment, address_book, &execute_clock);
+                                Self::commit_completed_execution(&mut processed_execution_environment, address_book, &mut execute_clock);
                                 // async-ly propagate the result to rpc subsystems
                                 to_next_stage.send(processed_execution_environment).unwrap();
                             }
@@ -1032,7 +1032,7 @@ impl ScheduleStage {
                         trace!("recv from execute: {:?}", processed_execution_environment.unique_weight);
                         executing_queue_count -= 1;
 
-                        Self::commit_completed_execution(&mut processed_execution_environment, address_book, &execute_clock);
+                        Self::commit_completed_execution(&mut processed_execution_environment, address_book, &mut execute_clock);
                         // async-ly propagate the result to rpc subsystems
                         to_next_stage.send(processed_execution_environment).unwrap();
                         if false && from_exec.len() > 0 {

From b6807d082dc28940cf834115b087e5f37f250445 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Wed, 24 Aug 2022 15:15:50 +0900
Subject: [PATCH 1257/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index dcd68a3d859afb..87706eb19fdfc4 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -904,7 +904,7 @@ impl ScheduleStage {
     fn commit_completed_execution(ee: &mut ExecutionEnvironment, address_book: &mut AddressBook, commit_time: &mut usize) {
         // do par()-ly?
 
-        info!("commit: seq: [{}st..{}.st; {}sd], queue: [{}qt..{}qt; {}qd] exec: [{}et..{}et; {}ed]", 
+        info!("commit: seq: [{}st..{}st; {}sd], queue: [{}qt..{}qt; {}qd] exec: [{}et..{}et; {}ed]", 
               ee.task.sequence_time(),
               ee.task.sequence_end_time(),
               ee.task.sequence_end_time() - ee.task.sequence_time(),

From 3d1e89fca368e51c73c0d75e01935eb63d8bee4c Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Wed, 24 Aug 2022 15:19:27 +0900
Subject: [PATCH 1258/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 87706eb19fdfc4..b296c5bd9d4b7f 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -910,7 +910,7 @@ impl ScheduleStage {
               ee.task.sequence_end_time() - ee.task.sequence_time(),
               ee.task.queue_time(), ee.task.queue_end_time(), ee.task.queue_end_time() - ee.task.queue_time(), 
               ee.task.execute_time(), *commit_time, *commit_time - ee.task.execute_time());
-        *commit_time = commit_time.checked_add(1).unwrap();
+        //*commit_time = commit_time.checked_add(1).unwrap();
 
         // which order for data race free?: unlocking / marking
         Self::unlock_after_execution(address_book, &mut ee.lock_attempts);

From 33d7c90b8156c82ed8a25ff18e09cc94ef25d2d9 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Wed, 24 Aug 2022 15:26:32 +0900
Subject: [PATCH 1259/3199] save

---
 ledger-tool/src/main.rs | 22 ++++++++++++++--------
 1 file changed, 14 insertions(+), 8 deletions(-)

diff --git a/ledger-tool/src/main.rs b/ledger-tool/src/main.rs
index 794003efe28bb8..79795111a20340 100644
--- a/ledger-tool/src/main.rs
+++ b/ledger-tool/src/main.rs
@@ -335,22 +335,28 @@ fn output_slot(
                         */
                         let uq = ee.unique_weight;
                         for mut lock_attempt in ee.lock_attempts.iter_mut() {
-                            let page = lock_attempt.target.page_ref();
-                            page.contended_unique_weights.remove_task_id(&uq);
-                            if let Some(mut task_cursor) = page.contended_unique_weights.heaviest_task_cursor() {
+                            let contended_unique_weights = lock_attempt.contended_unique_weights();
+                            contended_unique_weights.remove_task(&uq);
+                            contended_unique_weights.heaviest_task_cursor().map(|mut task_cursor| {
                                 let mut found = true;
-                                while !task_cursor.value().currently_contended() {
+                                assert_ne!(task_cursor.key(), &uq);
+                                let mut task = task_cursor.value();
+                                while !task.currently_contended() {
                                     if let Some(new_cursor) = task_cursor.prev() {
+                                        assert!(new_cursor.key() < task_cursor.key());
+                                        assert_ne!(new_cursor.key(), &uq);
                                         task_cursor = new_cursor;
+                                        task = task_cursor.value();
                                     } else {
                                         found = false;
                                         break;
                                     }
                                 }
-                                if found {
-                                    lock_attempt.heaviest_uncontended = Some(solana_scheduler::TaskInQueue::clone(task_cursor.value()));
-                                }
-                            }
+                                found.then(|| solana_scheduler::TaskInQueue::clone(task))
+                            }).flatten().map(|task| {
+                                lock_attempt.heaviest_uncontended = Some(task);
+                                ()
+                            });
                         }
                         post_execute_env_sender.send(ee).unwrap();
                     }

From 647b17297cfa22072a6bbfd0be0cec3f608e4c21 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Wed, 24 Aug 2022 15:26:46 +0900
Subject: [PATCH 1260/3199] save

---
 ledger-tool/src/main.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ledger-tool/src/main.rs b/ledger-tool/src/main.rs
index 79795111a20340..d49aee0186afa1 100644
--- a/ledger-tool/src/main.rs
+++ b/ledger-tool/src/main.rs
@@ -444,7 +444,7 @@ fn output_slot(
 
                                 let t = solana_scheduler::Task::new_for_queue(weight, tx);
                                 for lock_attempt in t.tx.1.iter() {
-                                    lock_attempt.target.page_ref().contended_unique_weights.insert_task_id(weight, solana_scheduler::TaskInQueue::clone(&t));
+                                    lock_attempt.contended_unique_weights().insert_task_id(weight, solana_scheduler::TaskInQueue::clone(&t));
                                 }
 
                                 muxed_sender

From 8fef81e4cc42f4531339d1a0bb219dfce1d034bc Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Wed, 24 Aug 2022 15:26:55 +0900
Subject: [PATCH 1261/3199] save

---
 ledger-tool/src/main.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ledger-tool/src/main.rs b/ledger-tool/src/main.rs
index d49aee0186afa1..17d8a44a50125f 100644
--- a/ledger-tool/src/main.rs
+++ b/ledger-tool/src/main.rs
@@ -444,7 +444,7 @@ fn output_slot(
 
                                 let t = solana_scheduler::Task::new_for_queue(weight, tx);
                                 for lock_attempt in t.tx.1.iter() {
-                                    lock_attempt.contended_unique_weights().insert_task_id(weight, solana_scheduler::TaskInQueue::clone(&t));
+                                    lock_attempt.contended_unique_weights().insert_task(weight, solana_scheduler::TaskInQueue::clone(&t));
                                 }
 
                                 muxed_sender

From cafea7c273798606fe1b01c997ba8a2c5f3e70cd Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Wed, 24 Aug 2022 15:31:00 +0900
Subject: [PATCH 1262/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index b296c5bd9d4b7f..c045a7c57c836a 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -846,7 +846,7 @@ impl ScheduleStage {
             let page = l.target.page_mut();
             if newly_uncontended && page.next_usage == Usage::Unused {
                 if let Some(task) = l.heaviest_uncontended.take() {
-                    assert!(!task.already_finished());
+                    //assert!(!task.already_finished());
                     if task.currently_contended() {
                         address_book.uncontended_task_ids.insert(task.unique_weight, task);
                     }

From 854457b07b343612003d3d9c91fec8babc24b46d Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Wed, 24 Aug 2022 15:46:24 +0900
Subject: [PATCH 1263/3199] save

---
 transaction-scheduler-bench/src/main.rs | 12 +++++++++---
 1 file changed, 9 insertions(+), 3 deletions(-)

diff --git a/transaction-scheduler-bench/src/main.rs b/transaction-scheduler-bench/src/main.rs
index bfae1fc63521d5..6d73f641eaa3ae 100644
--- a/transaction-scheduler-bench/src/main.rs
+++ b/transaction-scheduler-bench/src/main.rs
@@ -329,9 +329,15 @@ fn handle_transaction_batch(
         .num_transactions_scheduled
         .fetch_add(num_transactions as usize, Ordering::Relaxed);
 
-    sleep(Duration::from_micros(
-        num_transactions * execution_per_tx_us,
-    ));
+    if rand::thread_rng().gen_range(0..100) == 0 {
+        sleep(Duration::from_millis(
+            num_transactions * execution_per_tx_us,
+        ));
+    } else {
+        sleep(Duration::from_micros(
+            num_transactions * execution_per_tx_us,
+        ));
+    }
 
     let priority_collected = transaction_batch.task.tx.0.get_transaction_priority_details().unwrap().priority;
 

From 51298b8884827d734e23ba4c82764fb66d3cc789 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Wed, 24 Aug 2022 15:46:43 +0900
Subject: [PATCH 1264/3199] save

---
 transaction-scheduler-bench/src/main.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/transaction-scheduler-bench/src/main.rs b/transaction-scheduler-bench/src/main.rs
index 6d73f641eaa3ae..e6583add18f41e 100644
--- a/transaction-scheduler-bench/src/main.rs
+++ b/transaction-scheduler-bench/src/main.rs
@@ -330,8 +330,8 @@ fn handle_transaction_batch(
         .fetch_add(num_transactions as usize, Ordering::Relaxed);
 
     if rand::thread_rng().gen_range(0..100) == 0 {
-        sleep(Duration::from_millis(
-            num_transactions * execution_per_tx_us,
+        sleep(Duration::from_micros(
+            num_transactions * execution_per_tx_us * 50,
         ));
     } else {
         sleep(Duration::from_micros(

From c6b51d568951818bfe81e213742ee090aded10e5 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Wed, 24 Aug 2022 15:47:33 +0900
Subject: [PATCH 1265/3199] save

---
 transaction-scheduler-bench/src/main.rs | 1 +
 1 file changed, 1 insertion(+)

diff --git a/transaction-scheduler-bench/src/main.rs b/transaction-scheduler-bench/src/main.rs
index e6583add18f41e..33ffa67946cba9 100644
--- a/transaction-scheduler-bench/src/main.rs
+++ b/transaction-scheduler-bench/src/main.rs
@@ -330,6 +330,7 @@ fn handle_transaction_batch(
         .fetch_add(num_transactions as usize, Ordering::Relaxed);
 
     if rand::thread_rng().gen_range(0..100) == 0 {
+        panic!();
         sleep(Duration::from_micros(
             num_transactions * execution_per_tx_us * 50,
         ));

From 8dd5d09d7fff9f9d643bb7388c99c63887a5fa36 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Wed, 24 Aug 2022 15:48:01 +0900
Subject: [PATCH 1266/3199] save

---
 transaction-scheduler-bench/src/main.rs | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/transaction-scheduler-bench/src/main.rs b/transaction-scheduler-bench/src/main.rs
index 33ffa67946cba9..f0fa1c8459ab63 100644
--- a/transaction-scheduler-bench/src/main.rs
+++ b/transaction-scheduler-bench/src/main.rs
@@ -330,9 +330,8 @@ fn handle_transaction_batch(
         .fetch_add(num_transactions as usize, Ordering::Relaxed);
 
     if rand::thread_rng().gen_range(0..100) == 0 {
-        panic!();
         sleep(Duration::from_micros(
-            num_transactions * execution_per_tx_us * 50,
+            num_transactions * execution_per_tx_us * 5000,
         ));
     } else {
         sleep(Duration::from_micros(

From d1b46bce4803d187330bd4841c4eff9f1dd60d1a Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Wed, 24 Aug 2022 15:52:14 +0900
Subject: [PATCH 1267/3199] save

---
 transaction-scheduler-bench/src/main.rs | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/transaction-scheduler-bench/src/main.rs b/transaction-scheduler-bench/src/main.rs
index f0fa1c8459ab63..5944934bf3b25d 100644
--- a/transaction-scheduler-bench/src/main.rs
+++ b/transaction-scheduler-bench/src/main.rs
@@ -329,6 +329,7 @@ fn handle_transaction_batch(
         .num_transactions_scheduled
         .fetch_add(num_transactions as usize, Ordering::Relaxed);
 
+    /*
     if rand::thread_rng().gen_range(0..100) == 0 {
         sleep(Duration::from_micros(
             num_transactions * execution_per_tx_us * 5000,
@@ -338,6 +339,10 @@ fn handle_transaction_batch(
             num_transactions * execution_per_tx_us,
         ));
     }
+    */
+    sleep(Duration::from_micros(
+        rand::thread_rng().gen_range(0..1000)
+    ));
 
     let priority_collected = transaction_batch.task.tx.0.get_transaction_priority_details().unwrap().priority;
 

From 590d7fac7b114eb82a1a9f356cc2d14f44921f01 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Wed, 24 Aug 2022 15:52:42 +0900
Subject: [PATCH 1268/3199] save

---
 transaction-scheduler-bench/src/main.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/transaction-scheduler-bench/src/main.rs b/transaction-scheduler-bench/src/main.rs
index 5944934bf3b25d..901dbcda89ef83 100644
--- a/transaction-scheduler-bench/src/main.rs
+++ b/transaction-scheduler-bench/src/main.rs
@@ -341,7 +341,7 @@ fn handle_transaction_batch(
     }
     */
     sleep(Duration::from_micros(
-        rand::thread_rng().gen_range(0..1000)
+        rand::thread_rng().gen_range(0..execution_per_tx_us)
     ));
 
     let priority_collected = transaction_batch.task.tx.0.get_transaction_priority_details().unwrap().priority;

From a116235071bfbd0b688d62d8fd7f740b08e64dbb Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Wed, 24 Aug 2022 15:55:58 +0900
Subject: [PATCH 1269/3199] save

---
 scheduler/src/lib.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index c045a7c57c836a..82e9d6bcebe2f2 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -714,8 +714,6 @@ impl ScheduleStage {
             trace!("expediate pop from provisional queue [rest: {}]", address_book.fulfilled_provisional_task_ids.len());
             let next_task = unsafe { TaskInQueue::get_mut_unchecked(&mut a.1) };
 
-            *contended_count = contended_count.checked_sub(1).unwrap();
-            next_task.mark_as_uncontended();
             let lock_attempts = std::mem::take(&mut next_task.tx.1);
 
             return Some((a.0, a.1, lock_attempts));
@@ -780,6 +778,8 @@ impl ScheduleStage {
                     &unique_weight,
                     &mut next_task.tx.1,
                 );
+                *contended_count = contended_count.checked_sub(1).unwrap();
+                next_task.mark_as_uncontended();
                 drop(next_task);
                 address_book.provisioning_trackers.insert(unique_weight, (ProvisioningTracker::new(provisional_count), arc_next_task));
 

From c3a36aac04c20373c8b4e8fee51f56c438243820 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Wed, 24 Aug 2022 15:58:32 +0900
Subject: [PATCH 1270/3199] save

---
 scheduler/src/lib.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 82e9d6bcebe2f2..6eddb47cbdee42 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -996,7 +996,7 @@ impl ScheduleStage {
                         Multiplexed::FromPrevious(weighted_tx) => {
                             trace!("recv from previous");
 
-                            while false && from_exec.len() > 0 {
+                            while from_exec.len() > 0 {
                                 let mut processed_execution_environment = from_exec.recv().unwrap();
                                 trace!("recv from execute: {:?}", processed_execution_environment.unique_weight);
                                 executing_queue_count -= 1;
@@ -1035,7 +1035,7 @@ impl ScheduleStage {
                         Self::commit_completed_execution(&mut processed_execution_environment, address_book, &mut execute_clock);
                         // async-ly propagate the result to rpc subsystems
                         to_next_stage.send(processed_execution_environment).unwrap();
-                        if false && from_exec.len() > 0 {
+                        if from_exec.len() > 0 {
                             processed_execution_environment = from_exec.recv().unwrap();
                         } else {
                             break;

From 13c8bdb5567b4bd507fe788f0179c58084bf1952 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Wed, 24 Aug 2022 16:02:57 +0900
Subject: [PATCH 1271/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 6eddb47cbdee42..77404c8adfa025 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -904,7 +904,7 @@ impl ScheduleStage {
     fn commit_completed_execution(ee: &mut ExecutionEnvironment, address_book: &mut AddressBook, commit_time: &mut usize) {
         // do par()-ly?
 
-        info!("commit: seq: [{}st..{}st; {}sd], queue: [{}qt..{}qt; {}qd] exec: [{}et..{}et; {}ed]", 
+        trace!("commit: seq: [{}st..{}st; {}sd], queue: [{}qt..{}qt; {}qd] exec: [{}et..{}et; {}ed]", 
               ee.task.sequence_time(),
               ee.task.sequence_end_time(),
               ee.task.sequence_end_time() - ee.task.sequence_time(),

From b3e3d4d2d1668d977e5a41d0fb1da111bfd1a4d5 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Wed, 24 Aug 2022 16:04:12 +0900
Subject: [PATCH 1272/3199] save

---
 transaction-scheduler-bench/src/main.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/transaction-scheduler-bench/src/main.rs b/transaction-scheduler-bench/src/main.rs
index 901dbcda89ef83..a98c4c27a60756 100644
--- a/transaction-scheduler-bench/src/main.rs
+++ b/transaction-scheduler-bench/src/main.rs
@@ -341,7 +341,7 @@ fn handle_transaction_batch(
     }
     */
     sleep(Duration::from_micros(
-        rand::thread_rng().gen_range(0..execution_per_tx_us)
+        rand::thread_rng().gen_range(0..=execution_per_tx_us)
     ));
 
     let priority_collected = transaction_batch.task.tx.0.get_transaction_priority_details().unwrap().priority;

From be532736db1a14a70d003b17b9d112a6a3f037a5 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Wed, 24 Aug 2022 16:25:19 +0900
Subject: [PATCH 1273/3199] save

---
 transaction-scheduler-bench/src/main.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/transaction-scheduler-bench/src/main.rs b/transaction-scheduler-bench/src/main.rs
index a98c4c27a60756..9024ac871b01e0 100644
--- a/transaction-scheduler-bench/src/main.rs
+++ b/transaction-scheduler-bench/src/main.rs
@@ -131,7 +131,7 @@ fn spawn_unified_scheduler(
         let mut runnable_queue = solana_scheduler::TaskQueue::default();
 
         solana_scheduler::ScheduleStage::run(
-            num_execution_threads * 10,
+            num_execution_threads * 2,
             &mut runnable_queue,
             &mut address_book,
             &packet_batch_receiver.clone(),

From 828f3aaa7141071cde55a7b6eab70c0e2bd772aa Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Wed, 24 Aug 2022 16:55:20 +0900
Subject: [PATCH 1274/3199] save

---
 scheduler/src/lib.rs | 29 ++++++++++++++++++++++++++++-
 1 file changed, 28 insertions(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 77404c8adfa025..e11cff33122fbe 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -845,12 +845,39 @@ impl ScheduleStage {
 
             let page = l.target.page_mut();
             if newly_uncontended && page.next_usage == Usage::Unused {
+                let mut inserted = false;
+
                 if let Some(task) = l.heaviest_uncontended.take() {
                     //assert!(!task.already_finished());
                     if task.currently_contended() {
+                        inserted = true;
                         address_book.uncontended_task_ids.insert(task.unique_weight, task);
                     }
                 }
+
+                if !inserted {
+                    let contended_unique_weights = l.contended_unique_weights();
+                    contended_unique_weights.heaviest_task_cursor().map(|mut task_cursor| {
+                        let mut found = true;
+                        assert_ne!(task_cursor.key(), &uq);
+                        let mut task = task_cursor.value();
+                        while !task.currently_contended() {
+                            if let Some(new_cursor) = task_cursor.prev() {
+                                assert!(new_cursor.key() < task_cursor.key());
+                                assert_ne!(new_cursor.key(), &uq);
+                                task_cursor = new_cursor;
+                                task = task_cursor.value();
+                            } else {
+                                found = false;
+                                break;
+                            }
+                        }
+                        found.then(|| solana_scheduler::TaskInQueue::clone(task))
+                    }).flatten().map(|task| {
+                        address_book.uncontended_task_ids.insert(task.unique_weight, task);
+                        ()
+                    });
+                }
             }
             if page.current_usage == Usage::Unused && page.next_usage != Usage::Unused {
                 page.switch_to_next_usage();
@@ -904,7 +931,7 @@ impl ScheduleStage {
     fn commit_completed_execution(ee: &mut ExecutionEnvironment, address_book: &mut AddressBook, commit_time: &mut usize) {
         // do par()-ly?
 
-        trace!("commit: seq: [{}st..{}st; {}sd], queue: [{}qt..{}qt; {}qd] exec: [{}et..{}et; {}ed]", 
+        trace!("commit: seq: [{}sT..{}sT; {}sD], queue: [{}qT..{}qT; {}qD] exec: [{}eT..{}eT; {}eD]", 
               ee.task.sequence_time(),
               ee.task.sequence_end_time(),
               ee.task.sequence_end_time() - ee.task.sequence_time(),

From 3ceff6508b618812ab06df94a6c6eb8a9ce1c44d Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Wed, 24 Aug 2022 16:55:46 +0900
Subject: [PATCH 1275/3199] save

---
 scheduler/src/lib.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index e11cff33122fbe..04d87efbe82a2d 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -859,12 +859,12 @@ impl ScheduleStage {
                     let contended_unique_weights = l.contended_unique_weights();
                     contended_unique_weights.heaviest_task_cursor().map(|mut task_cursor| {
                         let mut found = true;
-                        assert_ne!(task_cursor.key(), &uq);
+                        assert_ne!(task_cursor.key(), &task.unique_weight);
                         let mut task = task_cursor.value();
                         while !task.currently_contended() {
                             if let Some(new_cursor) = task_cursor.prev() {
                                 assert!(new_cursor.key() < task_cursor.key());
-                                assert_ne!(new_cursor.key(), &uq);
+                                assert_ne!(new_cursor.key(), &task.unique_weight);
                                 task_cursor = new_cursor;
                                 task = task_cursor.value();
                             } else {

From 9159252d078e9a593ae6b6b6617d753ad636fe16 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Wed, 24 Aug 2022 16:56:01 +0900
Subject: [PATCH 1276/3199] save

---
 scheduler/src/lib.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 04d87efbe82a2d..273ae7bb571a60 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -859,12 +859,12 @@ impl ScheduleStage {
                     let contended_unique_weights = l.contended_unique_weights();
                     contended_unique_weights.heaviest_task_cursor().map(|mut task_cursor| {
                         let mut found = true;
-                        assert_ne!(task_cursor.key(), &task.unique_weight);
+                        //assert_ne!(task_cursor.key(), &task.uq);
                         let mut task = task_cursor.value();
                         while !task.currently_contended() {
                             if let Some(new_cursor) = task_cursor.prev() {
                                 assert!(new_cursor.key() < task_cursor.key());
-                                assert_ne!(new_cursor.key(), &task.unique_weight);
+                                //assert_ne!(new_cursor.key(), &uq);
                                 task_cursor = new_cursor;
                                 task = task_cursor.value();
                             } else {

From 564653b3b1a98df976ae8f0e55282f21f1487cf6 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Wed, 24 Aug 2022 16:56:15 +0900
Subject: [PATCH 1277/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 273ae7bb571a60..45d17929a9266b 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -872,7 +872,7 @@ impl ScheduleStage {
                                 break;
                             }
                         }
-                        found.then(|| solana_scheduler::TaskInQueue::clone(task))
+                        found.then(|| TaskInQueue::clone(task))
                     }).flatten().map(|task| {
                         address_book.uncontended_task_ids.insert(task.unique_weight, task);
                         ()

From 9483a451cb4cda9efe379578c867ac084dc436b3 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Wed, 24 Aug 2022 16:58:49 +0900
Subject: [PATCH 1278/3199] save

---
 scheduler/src/lib.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 45d17929a9266b..3433116c0a1e48 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -843,8 +843,7 @@ impl ScheduleStage {
         for mut l in lock_attempts.into_iter() {
             let newly_uncontended = address_book.reset_lock(&mut l, true);
 
-            let page = l.target.page_mut();
-            if newly_uncontended && page.next_usage == Usage::Unused {
+            if newly_uncontended && l.target.page_ref().next_usage == Usage::Unused {
                 let mut inserted = false;
 
                 if let Some(task) = l.heaviest_uncontended.take() {
@@ -879,6 +878,7 @@ impl ScheduleStage {
                     });
                 }
             }
+            let page = l.target.page_mut();
             if page.current_usage == Usage::Unused && page.next_usage != Usage::Unused {
                 page.switch_to_next_usage();
                 for task_id in std::mem::take(&mut page.provisional_task_ids).into_iter() {

From ee29d313337e1fbbef48ff787dd6c3d9f818cbb3 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Wed, 24 Aug 2022 16:59:58 +0900
Subject: [PATCH 1279/3199] save

---
 scheduler/src/lib.rs | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 3433116c0a1e48..18522e146a83ef 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -843,7 +843,8 @@ impl ScheduleStage {
         for mut l in lock_attempts.into_iter() {
             let newly_uncontended = address_book.reset_lock(&mut l, true);
 
-            if newly_uncontended && l.target.page_ref().next_usage == Usage::Unused {
+            let page = l.target.page_mut();
+            if newly_uncontended && page.next_usage == Usage::Unused {
                 let mut inserted = false;
 
                 if let Some(task) = l.heaviest_uncontended.take() {
@@ -855,7 +856,7 @@ impl ScheduleStage {
                 }
 
                 if !inserted {
-                    let contended_unique_weights = l.contended_unique_weights();
+                    let contended_unique_weights = page.contended_unique_weights();
                     contended_unique_weights.heaviest_task_cursor().map(|mut task_cursor| {
                         let mut found = true;
                         //assert_ne!(task_cursor.key(), &task.uq);
@@ -878,7 +879,6 @@ impl ScheduleStage {
                     });
                 }
             }
-            let page = l.target.page_mut();
             if page.current_usage == Usage::Unused && page.next_usage != Usage::Unused {
                 page.switch_to_next_usage();
                 for task_id in std::mem::take(&mut page.provisional_task_ids).into_iter() {

From 19d456b5a501b798b1faed6050b90ea4b78ee525 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Wed, 24 Aug 2022 17:00:07 +0900
Subject: [PATCH 1280/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 18522e146a83ef..7555ff4442b0fc 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -856,7 +856,7 @@ impl ScheduleStage {
                 }
 
                 if !inserted {
-                    let contended_unique_weights = page.contended_unique_weights();
+                    let contended_unique_weights = page.contended_unique_weights;
                     contended_unique_weights.heaviest_task_cursor().map(|mut task_cursor| {
                         let mut found = true;
                         //assert_ne!(task_cursor.key(), &task.uq);

From 0fbeab8c6f9df20159571b51d20421ee09f1997c Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Wed, 24 Aug 2022 17:00:14 +0900
Subject: [PATCH 1281/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 7555ff4442b0fc..c2b42aabeabdd5 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -856,7 +856,7 @@ impl ScheduleStage {
                 }
 
                 if !inserted {
-                    let contended_unique_weights = page.contended_unique_weights;
+                    let contended_unique_weights = &page.contended_unique_weights;
                     contended_unique_weights.heaviest_task_cursor().map(|mut task_cursor| {
                         let mut found = true;
                         //assert_ne!(task_cursor.key(), &task.uq);

From a2aff6fc8c1ecaa63b7f784c74232c4e97a9c3e8 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Wed, 24 Aug 2022 21:19:40 +0900
Subject: [PATCH 1282/3199] save

---
 scheduler/src/lib.rs | 28 +++++++++++++++++++++-------
 1 file changed, 21 insertions(+), 7 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index c2b42aabeabdd5..50dea38bd34ac7 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -439,7 +439,7 @@ pub struct Task {
 
 impl Task {
     pub fn new_for_queue(unique_weight: UniqueWeight, tx: Box<(SanitizedTransaction, Vec<LockAttempt>)>) -> std::sync::Arc<Self> {
-        TaskInQueue::new(Self { unique_weight, tx, contention_count: 0, uncontended: Default::default(), sequence_time: std::sync::atomic::AtomicUsize::new(usize::max_value()), sequence_end_time: std::sync::atomic::AtomicUsize::new(usize::max_value()), queue_time: std::sync::atomic::AtomicUsize::new(usize::max_value()), queue_end_time: std::sync::atomic::AtomicUsize::new(usize::max_value()), execute_time: std::sync::atomic::AtomicUsize::new(usize::max_value()) })
+        TaskInQueue::new(Self { unique_weight, tx, contention_count: 0, uncontended: Default::default(), sequence_time: std::sync::atomic::AtomicUsize::new(usize::max_value()), sequence_end_time: std::sync::atomic::AtomicUsize::new(usize::max_value()), queue_time: std::sync::atomic::AtomicUsize::new(usize::max_value()), queue_end_time: std::sync::atomic::AtomicUsize::new(usize::max_value()), execute_time: std::sync::atomic::AtomicUsize::new(usize::max_value()), commit_time: std::sync::atomic::AtomicUsize::new(usize::max_value()) })
     }
 
     pub fn record_sequence_time(&self, clock: usize) {
@@ -476,6 +476,23 @@ impl Task {
         self.execute_time.load(std::sync::atomic::Ordering::SeqCst)
     }
 
+    pub fn record_commit_time(&self, execute_clock: usize) {
+        self.commit_time.store(execute_clock, std::sync::atomic::Ordering::SeqCst);
+    }
+
+    pub fn commit_time(&self) -> usize {
+        self.commit_time.load(std::sync::atomic::Ordering::SeqCst)
+    }
+
+    pub fn trace_timestamps(&self) {
+        trace!("commit: seq: [{}sT..{}sT; {}sD], queue: [{}qT..{}qT; {}qD] exec: [{}eT..{}eT; {}eD]", 
+              self.sequence_time(),
+              self.sequence_end_time(),
+              self.sequence_end_time() - self.sequence_time(),
+              self.queue_time(), self.queue_end_time(), self.queue_end_time() - self.queue_time(), 
+              self.execute_time(), self.commit_time(), self.commit_time() - self.execute_time());
+    }
+
 
     pub fn clone_for_test(&self) -> Self {
         Self {
@@ -488,6 +505,7 @@ impl Task {
             queue_time: std::sync::atomic::AtomicUsize::new(usize::max_value()),
             queue_end_time: std::sync::atomic::AtomicUsize::new(usize::max_value()),
             execute_time: std::sync::atomic::AtomicUsize::new(usize::max_value()),
+            commit_time: std::sync::atomic::AtomicUsize::new(usize::max_value()),
         }
     }
 
@@ -931,12 +949,8 @@ impl ScheduleStage {
     fn commit_completed_execution(ee: &mut ExecutionEnvironment, address_book: &mut AddressBook, commit_time: &mut usize) {
         // do par()-ly?
 
-        trace!("commit: seq: [{}sT..{}sT; {}sD], queue: [{}qT..{}qT; {}qD] exec: [{}eT..{}eT; {}eD]", 
-              ee.task.sequence_time(),
-              ee.task.sequence_end_time(),
-              ee.task.sequence_end_time() - ee.task.sequence_time(),
-              ee.task.queue_time(), ee.task.queue_end_time(), ee.task.queue_end_time() - ee.task.queue_time(), 
-              ee.task.execute_time(), *commit_time, *commit_time - ee.task.execute_time());
+        ee.task.record_commit_time(commit_time);
+        ee.task.trace_timestamps();
         //*commit_time = commit_time.checked_add(1).unwrap();
 
         // which order for data race free?: unlocking / marking

From 0eed7b62ef88b252b649198c57cd9654dde2e76a Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Wed, 24 Aug 2022 21:20:00 +0900
Subject: [PATCH 1283/3199] save

---
 scheduler/src/lib.rs | 1 +
 1 file changed, 1 insertion(+)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 50dea38bd34ac7..04543438abea4a 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -430,6 +430,7 @@ pub struct Task {
     pub queue_time: std::sync::atomic::AtomicUsize,
     pub queue_end_time: std::sync::atomic::AtomicUsize,
     pub execute_time: std::sync::atomic::AtomicUsize,
+    pub commit_time: std::sync::atomic::AtomicUsize,
 }
 
 // sequence_time -> seq clock

From b823c17b97c85647bddee682d7f805a35cfbe490 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Wed, 24 Aug 2022 21:20:09 +0900
Subject: [PATCH 1284/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 04543438abea4a..bb684205f3ce79 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -950,7 +950,7 @@ impl ScheduleStage {
     fn commit_completed_execution(ee: &mut ExecutionEnvironment, address_book: &mut AddressBook, commit_time: &mut usize) {
         // do par()-ly?
 
-        ee.task.record_commit_time(commit_time);
+        ee.task.record_commit_time(*commit_time);
         ee.task.trace_timestamps();
         //*commit_time = commit_time.checked_add(1).unwrap();
 

From 0a7911861e7d3e3c9c11940ef9ee9c3509bddb9e Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Wed, 24 Aug 2022 21:21:47 +0900
Subject: [PATCH 1285/3199] save

---
 scheduler/src/lib.rs                    | 7 ++++---
 transaction-scheduler-bench/src/main.rs | 1 +
 2 files changed, 5 insertions(+), 3 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index bb684205f3ce79..62c5a26c8eb05d 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -485,8 +485,9 @@ impl Task {
         self.commit_time.load(std::sync::atomic::Ordering::SeqCst)
     }
 
-    pub fn trace_timestamps(&self) {
-        trace!("commit: seq: [{}sT..{}sT; {}sD], queue: [{}qT..{}qT; {}qD] exec: [{}eT..{}eT; {}eD]", 
+    pub fn trace_timestamps(&self, prefix: &str) {
+        trace!("{}: seq: [{}sT..{}sT; {}sD], queue: [{}qT..{}qT; {}qD] exec: [{}eT..{}eT; {}eD]", 
+               prefix,
               self.sequence_time(),
               self.sequence_end_time(),
               self.sequence_end_time() - self.sequence_time(),
@@ -951,7 +952,7 @@ impl ScheduleStage {
         // do par()-ly?
 
         ee.task.record_commit_time(*commit_time);
-        ee.task.trace_timestamps();
+        ee.task.trace_timestamps("commit");
         //*commit_time = commit_time.checked_add(1).unwrap();
 
         // which order for data race free?: unlocking / marking
diff --git a/transaction-scheduler-bench/src/main.rs b/transaction-scheduler-bench/src/main.rs
index 9024ac871b01e0..1cf2f3afecf884 100644
--- a/transaction-scheduler-bench/src/main.rs
+++ b/transaction-scheduler-bench/src/main.rs
@@ -354,6 +354,7 @@ fn handle_transaction_batch(
         .fetch_add(priority_collected, Ordering::Relaxed);
 
     let uq = transaction_batch.unique_weight;
+    transaction_batch.task.trace_timestamps();
     for mut lock_attempt in transaction_batch.lock_attempts.iter_mut() {
         let contended_unique_weights = lock_attempt.contended_unique_weights();
         contended_unique_weights.remove_task(&uq);

From 62d5fbaa2dc7a5e7f32afc87a98d831fd3a82a1f Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Wed, 24 Aug 2022 21:21:58 +0900
Subject: [PATCH 1286/3199] save

---
 transaction-scheduler-bench/src/main.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/transaction-scheduler-bench/src/main.rs b/transaction-scheduler-bench/src/main.rs
index 1cf2f3afecf884..b3f24b81263d38 100644
--- a/transaction-scheduler-bench/src/main.rs
+++ b/transaction-scheduler-bench/src/main.rs
@@ -354,7 +354,7 @@ fn handle_transaction_batch(
         .fetch_add(priority_collected, Ordering::Relaxed);
 
     let uq = transaction_batch.unique_weight;
-    transaction_batch.task.trace_timestamps();
+    transaction_batch.task.trace_timestamps("in_exec");
     for mut lock_attempt in transaction_batch.lock_attempts.iter_mut() {
         let contended_unique_weights = lock_attempt.contended_unique_weights();
         contended_unique_weights.remove_task(&uq);

From aed09660ac8f1b0031e6677760889d387470794b Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Wed, 24 Aug 2022 21:24:32 +0900
Subject: [PATCH 1287/3199] save

---
 transaction-scheduler-bench/src/main.rs | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/transaction-scheduler-bench/src/main.rs b/transaction-scheduler-bench/src/main.rs
index b3f24b81263d38..55549f0a5f59f9 100644
--- a/transaction-scheduler-bench/src/main.rs
+++ b/transaction-scheduler-bench/src/main.rs
@@ -354,7 +354,7 @@ fn handle_transaction_batch(
         .fetch_add(priority_collected, Ordering::Relaxed);
 
     let uq = transaction_batch.unique_weight;
-    transaction_batch.task.trace_timestamps("in_exec");
+    transaction_batch.task.trace_timestamps("in_exec(self)");
     for mut lock_attempt in transaction_batch.lock_attempts.iter_mut() {
         let contended_unique_weights = lock_attempt.contended_unique_weights();
         contended_unique_weights.remove_task(&uq);
@@ -362,12 +362,14 @@ fn handle_transaction_batch(
             let mut found = true;
             assert_ne!(task_cursor.key(), &uq);
             let mut task = task_cursor.value();
+            task.trace_timestamps("in_exec(initial list)");
             while !task.currently_contended() {
                 if let Some(new_cursor) = task_cursor.prev() {
                     assert!(new_cursor.key() < task_cursor.key());
                     assert_ne!(new_cursor.key(), &uq);
                     task_cursor = new_cursor;
                     task = task_cursor.value();
+                    task.trace_timestamps("in_exec(subsequent list)");
                 } else {
                     found = false;
                     break;

From dc876d93c630c3e2ca10e32c3a0bbf77320a055c Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Wed, 24 Aug 2022 21:25:27 +0900
Subject: [PATCH 1288/3199] save

---
 scheduler/src/lib.rs | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 62c5a26c8eb05d..767fe85fc25b6e 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -486,8 +486,9 @@ impl Task {
     }
 
     pub fn trace_timestamps(&self, prefix: &str) {
-        trace!("{}: seq: [{}sT..{}sT; {}sD], queue: [{}qT..{}qT; {}qD] exec: [{}eT..{}eT; {}eD]", 
-               prefix,
+        trace!("{}: {:016x} seq: [{}sT..{}sT; {}sD], queue: [{}qT..{}qT; {}qD] exec: [{}eT..{}eT; {}eD]", 
+              prefix,
+              self.unique_weight,
               self.sequence_time(),
               self.sequence_end_time(),
               self.sequence_end_time() - self.sequence_time(),

From 60858239a20bf57a134f53a7b27d29336daeab4c Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Wed, 24 Aug 2022 21:27:00 +0900
Subject: [PATCH 1289/3199] save

---
 transaction-scheduler-bench/src/main.rs | 1 +
 1 file changed, 1 insertion(+)

diff --git a/transaction-scheduler-bench/src/main.rs b/transaction-scheduler-bench/src/main.rs
index 55549f0a5f59f9..b8dd685d3274ec 100644
--- a/transaction-scheduler-bench/src/main.rs
+++ b/transaction-scheduler-bench/src/main.rs
@@ -377,6 +377,7 @@ fn handle_transaction_batch(
             }
             found.then(|| solana_scheduler::TaskInQueue::clone(task))
         }).flatten().map(|task| {
+            task.trace_timestamps("in_exec(heaviest)");
             lock_attempt.heaviest_uncontended = Some(task);
             ()
         });

From beccc8f610b32e25d9fdcffb763c12b68a8302a5 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Wed, 24 Aug 2022 21:28:03 +0900
Subject: [PATCH 1290/3199] save

---
 scheduler/src/lib.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 767fe85fc25b6e..ee5f1472d1081f 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -876,7 +876,7 @@ impl ScheduleStage {
                     }
                 }
 
-                if !inserted {
+                /*if !inserted {
                     let contended_unique_weights = &page.contended_unique_weights;
                     contended_unique_weights.heaviest_task_cursor().map(|mut task_cursor| {
                         let mut found = true;
@@ -898,7 +898,7 @@ impl ScheduleStage {
                         address_book.uncontended_task_ids.insert(task.unique_weight, task);
                         ()
                     });
-                }
+                }*/
             }
             if page.current_usage == Usage::Unused && page.next_usage != Usage::Unused {
                 page.switch_to_next_usage();

From 5fdf017cd8f1db0a4f5fc73856d96537608a1bc4 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Wed, 24 Aug 2022 21:36:21 +0900
Subject: [PATCH 1291/3199] save

---
 scheduler/src/lib.rs                    | 4 ++++
 transaction-scheduler-bench/src/main.rs | 2 +-
 2 files changed, 5 insertions(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index ee5f1472d1081f..b64f541e9bc964 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -512,6 +512,10 @@ impl Task {
         }
     }
 
+    pub fn newly_queued(&self) -> bool {
+        self.uncontended.load(std::sync::atomic::Ordering::SeqCst) == 0
+    }
+
     pub fn currently_contended(&self) -> bool {
         self.uncontended.load(std::sync::atomic::Ordering::SeqCst) == 1
     }
diff --git a/transaction-scheduler-bench/src/main.rs b/transaction-scheduler-bench/src/main.rs
index b8dd685d3274ec..07552a4520dc72 100644
--- a/transaction-scheduler-bench/src/main.rs
+++ b/transaction-scheduler-bench/src/main.rs
@@ -363,7 +363,7 @@ fn handle_transaction_batch(
             assert_ne!(task_cursor.key(), &uq);
             let mut task = task_cursor.value();
             task.trace_timestamps("in_exec(initial list)");
-            while !task.currently_contended() {
+            while !(task.currently_contended() || task.newly_queued()) {
                 if let Some(new_cursor) = task_cursor.prev() {
                     assert!(new_cursor.key() < task_cursor.key());
                     assert_ne!(new_cursor.key(), &uq);

From 9777e59638b4feeb55ca90a1861fbb888271b53f Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Wed, 24 Aug 2022 21:37:03 +0900
Subject: [PATCH 1292/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index b64f541e9bc964..59c3c32a64c353 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -874,7 +874,7 @@ impl ScheduleStage {
 
                 if let Some(task) = l.heaviest_uncontended.take() {
                     //assert!(!task.already_finished());
-                    if task.currently_contended() {
+                    if task.currently_contended() || task.newly_queued() {
                         inserted = true;
                         address_book.uncontended_task_ids.insert(task.unique_weight, task);
                     }

From 4cc50ee027f8b5e5ce53f6a97673c495cb3cc4a1 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Wed, 24 Aug 2022 21:39:07 +0900
Subject: [PATCH 1293/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 59c3c32a64c353..fddf0462e6c25a 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -529,7 +529,7 @@ impl Task {
     }
 
     fn mark_as_uncontended(&self) {
-        assert!(self.currently_contended());
+        //assert!(self.currently_contended());
         self.uncontended.store(2, std::sync::atomic::Ordering::SeqCst)
     }
 

From 449689abc51b5589e98bc9aca4186a840d331c10 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Wed, 24 Aug 2022 21:40:02 +0900
Subject: [PATCH 1294/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index fddf0462e6c25a..0e3c4abb5b76af 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -814,7 +814,7 @@ impl ScheduleStage {
 
             trace!("successful lock: (from_runnable: {}) after {} contentions", from_runnable, next_task.contention_count);
 
-            if !from_runnable {
+            if next_task.currently_contended() {
                 *contended_count = contended_count.checked_sub(1).unwrap();
                 next_task.mark_as_uncontended();
             }

From e12a1f47b94f3f56e8f5124f461f4f94917b43b4 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Wed, 24 Aug 2022 21:51:32 +0900
Subject: [PATCH 1295/3199] save

---
 scheduler/src/lib.rs | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 0e3c4abb5b76af..294a34ddc43f73 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -748,7 +748,11 @@ impl ScheduleStage {
         loop {
         if let Some((from_runnable, mut arc_next_task)) = Self::select_next_task(runnable_queue, address_book) {
             trace!("pop loop iteration");
-            if from_runnable {
+            if from_runnable && arc_next_task.queue_time() != usize::max_value() {
+                continue;
+            }
+
+            if arc_next_task.queue_time() == usize::max_value() {
                 arc_next_task.record_queue_time(*sequence_clock, *queue_clock);
                 *queue_clock = queue_clock.checked_add(1).unwrap();
             }
@@ -876,6 +880,7 @@ impl ScheduleStage {
                     //assert!(!task.already_finished());
                     if task.currently_contended() || task.newly_queued() {
                         inserted = true;
+                        task.mark_as_contended();
                         address_book.uncontended_task_ids.insert(task.unique_weight, task);
                     }
                 }

From f9faa1c3f0ed7ede672d0661b16ec1088baaf8c3 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Wed, 24 Aug 2022 21:52:04 +0900
Subject: [PATCH 1296/3199] save

---
 scheduler/src/lib.rs | 1 -
 1 file changed, 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 294a34ddc43f73..2828f1456f9ec4 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -880,7 +880,6 @@ impl ScheduleStage {
                     //assert!(!task.already_finished());
                     if task.currently_contended() || task.newly_queued() {
                         inserted = true;
-                        task.mark_as_contended();
                         address_book.uncontended_task_ids.insert(task.unique_weight, task);
                     }
                 }

From 7868487a809f9d23fc9e27ba107ee3201bb63373 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Wed, 24 Aug 2022 21:52:41 +0900
Subject: [PATCH 1297/3199] save

---
 scheduler/src/lib.rs | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 2828f1456f9ec4..5031be0a37fcac 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -534,7 +534,8 @@ impl Task {
     }
 
     fn mark_as_finished(&self) {
-        assert!(!self.already_finished() && !self.currently_contended());
+        assert!(!self.already_finished());
+        assert!(!self.currently_contended());
         self.uncontended.store(3, std::sync::atomic::Ordering::SeqCst)
     }
 }

From 37441560e3ab5210a0912d547fd72c7570f85ed1 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Wed, 24 Aug 2022 21:55:06 +0900
Subject: [PATCH 1298/3199] save

---
 scheduler/src/lib.rs | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 5031be0a37fcac..dcefb5f1e40edb 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -752,6 +752,9 @@ impl ScheduleStage {
             if from_runnable && arc_next_task.queue_time() != usize::max_value() {
                 continue;
             }
+            if arc_next_task.already_finished() {
+                continue;
+            }
 
             if arc_next_task.queue_time() == usize::max_value() {
                 arc_next_task.record_queue_time(*sequence_clock, *queue_clock);

From 430a755325f0ee12bd0835a88a23b154dac9f00e Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Wed, 24 Aug 2022 21:58:33 +0900
Subject: [PATCH 1299/3199] save

---
 scheduler/src/lib.rs | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index dcefb5f1e40edb..8b5a2a190cf293 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -739,6 +739,9 @@ impl ScheduleStage {
         if let Some(mut a) = address_book.fulfilled_provisional_task_ids.pop_last() {
             trace!("expediate pop from provisional queue [rest: {}]", address_book.fulfilled_provisional_task_ids.len());
             let next_task = unsafe { TaskInQueue::get_mut_unchecked(&mut a.1) };
+            if next_task.already_finished() {
+                continue;
+            }
 
             let lock_attempts = std::mem::take(&mut next_task.tx.1);
 

From 82f7ff8ae659f735b8bd3b2c17e123d5a8fc10eb Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Wed, 24 Aug 2022 21:58:48 +0900
Subject: [PATCH 1300/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 8b5a2a190cf293..b538d1a3798c72 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -740,7 +740,7 @@ impl ScheduleStage {
             trace!("expediate pop from provisional queue [rest: {}]", address_book.fulfilled_provisional_task_ids.len());
             let next_task = unsafe { TaskInQueue::get_mut_unchecked(&mut a.1) };
             if next_task.already_finished() {
-                continue;
+                return None;
             }
 
             let lock_attempts = std::mem::take(&mut next_task.tx.1);

From f4348d4567fe210901040dc24504b36ddbe31cbe Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Wed, 24 Aug 2022 22:02:00 +0900
Subject: [PATCH 1301/3199] save

---
 scheduler/src/lib.rs                    | 25 +++++--------------------
 transaction-scheduler-bench/src/main.rs |  2 +-
 2 files changed, 6 insertions(+), 21 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index b538d1a3798c72..ee5f1472d1081f 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -512,10 +512,6 @@ impl Task {
         }
     }
 
-    pub fn newly_queued(&self) -> bool {
-        self.uncontended.load(std::sync::atomic::Ordering::SeqCst) == 0
-    }
-
     pub fn currently_contended(&self) -> bool {
         self.uncontended.load(std::sync::atomic::Ordering::SeqCst) == 1
     }
@@ -529,13 +525,12 @@ impl Task {
     }
 
     fn mark_as_uncontended(&self) {
-        //assert!(self.currently_contended());
+        assert!(self.currently_contended());
         self.uncontended.store(2, std::sync::atomic::Ordering::SeqCst)
     }
 
     fn mark_as_finished(&self) {
-        assert!(!self.already_finished());
-        assert!(!self.currently_contended());
+        assert!(!self.already_finished() && !self.currently_contended());
         self.uncontended.store(3, std::sync::atomic::Ordering::SeqCst)
     }
 }
@@ -739,9 +734,6 @@ impl ScheduleStage {
         if let Some(mut a) = address_book.fulfilled_provisional_task_ids.pop_last() {
             trace!("expediate pop from provisional queue [rest: {}]", address_book.fulfilled_provisional_task_ids.len());
             let next_task = unsafe { TaskInQueue::get_mut_unchecked(&mut a.1) };
-            if next_task.already_finished() {
-                return None;
-            }
 
             let lock_attempts = std::mem::take(&mut next_task.tx.1);
 
@@ -752,14 +744,7 @@ impl ScheduleStage {
         loop {
         if let Some((from_runnable, mut arc_next_task)) = Self::select_next_task(runnable_queue, address_book) {
             trace!("pop loop iteration");
-            if from_runnable && arc_next_task.queue_time() != usize::max_value() {
-                continue;
-            }
-            if arc_next_task.already_finished() {
-                continue;
-            }
-
-            if arc_next_task.queue_time() == usize::max_value() {
+            if from_runnable {
                 arc_next_task.record_queue_time(*sequence_clock, *queue_clock);
                 *queue_clock = queue_clock.checked_add(1).unwrap();
             }
@@ -825,7 +810,7 @@ impl ScheduleStage {
 
             trace!("successful lock: (from_runnable: {}) after {} contentions", from_runnable, next_task.contention_count);
 
-            if next_task.currently_contended() {
+            if !from_runnable {
                 *contended_count = contended_count.checked_sub(1).unwrap();
                 next_task.mark_as_uncontended();
             }
@@ -885,7 +870,7 @@ impl ScheduleStage {
 
                 if let Some(task) = l.heaviest_uncontended.take() {
                     //assert!(!task.already_finished());
-                    if task.currently_contended() || task.newly_queued() {
+                    if task.currently_contended() {
                         inserted = true;
                         address_book.uncontended_task_ids.insert(task.unique_weight, task);
                     }
diff --git a/transaction-scheduler-bench/src/main.rs b/transaction-scheduler-bench/src/main.rs
index 07552a4520dc72..b8dd685d3274ec 100644
--- a/transaction-scheduler-bench/src/main.rs
+++ b/transaction-scheduler-bench/src/main.rs
@@ -363,7 +363,7 @@ fn handle_transaction_batch(
             assert_ne!(task_cursor.key(), &uq);
             let mut task = task_cursor.value();
             task.trace_timestamps("in_exec(initial list)");
-            while !(task.currently_contended() || task.newly_queued()) {
+            while !task.currently_contended() {
                 if let Some(new_cursor) = task_cursor.prev() {
                     assert!(new_cursor.key() < task_cursor.key());
                     assert_ne!(new_cursor.key(), &uq);

From 85092605c55cfedbb665005f5cf1eaaae48a46f9 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Wed, 24 Aug 2022 22:05:38 +0900
Subject: [PATCH 1302/3199] save

---
 scheduler/src/lib.rs | 21 +++++++++++----------
 1 file changed, 11 insertions(+), 10 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index ee5f1472d1081f..c849d0d1761f24 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1040,16 +1040,6 @@ impl ScheduleStage {
                         Multiplexed::FromPrevious(weighted_tx) => {
                             trace!("recv from previous");
 
-                            while from_exec.len() > 0 {
-                                let mut processed_execution_environment = from_exec.recv().unwrap();
-                                trace!("recv from execute: {:?}", processed_execution_environment.unique_weight);
-                                executing_queue_count -= 1;
-
-                                Self::commit_completed_execution(&mut processed_execution_environment, address_book, &mut execute_clock);
-                                // async-ly propagate the result to rpc subsystems
-                                to_next_stage.send(processed_execution_environment).unwrap();
-                            }
-
                             Self::register_runnable_task(weighted_tx, runnable_queue, &mut current_unique_key, &mut sequence_time);
 
                             while from.len() > 0 && from_exec.len() == 0 {
@@ -1064,6 +1054,17 @@ impl ScheduleStage {
                                     }
                                 }
                             }
+
+                            while from_exec.len() > 0 {
+                                let mut processed_execution_environment = from_exec.recv().unwrap();
+                                trace!("recv from execute: {:?}", processed_execution_environment.unique_weight);
+                                executing_queue_count -= 1;
+
+                                Self::commit_completed_execution(&mut processed_execution_environment, address_book, &mut execute_clock);
+                                // async-ly propagate the result to rpc subsystems
+                                to_next_stage.send(processed_execution_environment).unwrap();
+                            }
+
                         }
                         Multiplexed::FromPreviousBatched(vvv) => {
                             unreachable!();

From 0c2e15c23880e9645b61e198aa4d4e879a39c3a9 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Wed, 24 Aug 2022 22:06:42 +0900
Subject: [PATCH 1303/3199] save

---
 scheduler/src/lib.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index c849d0d1761f24..bdc9ff8eefc0be 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1055,7 +1055,7 @@ impl ScheduleStage {
                                 }
                             }
 
-                            while from_exec.len() > 0 {
+                            while false && from_exec.len() > 0 {
                                 let mut processed_execution_environment = from_exec.recv().unwrap();
                                 trace!("recv from execute: {:?}", processed_execution_environment.unique_weight);
                                 executing_queue_count -= 1;
@@ -1080,7 +1080,7 @@ impl ScheduleStage {
                         Self::commit_completed_execution(&mut processed_execution_environment, address_book, &mut execute_clock);
                         // async-ly propagate the result to rpc subsystems
                         to_next_stage.send(processed_execution_environment).unwrap();
-                        if from_exec.len() > 0 {
+                        if false && from_exec.len() > 0 {
                             processed_execution_environment = from_exec.recv().unwrap();
                         } else {
                             break;

From 4d60af0f4b5ebba22bd66df0ea8716921d85e76e Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Wed, 24 Aug 2022 22:09:08 +0900
Subject: [PATCH 1304/3199] save

---
 transaction-scheduler-bench/src/main.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/transaction-scheduler-bench/src/main.rs b/transaction-scheduler-bench/src/main.rs
index b8dd685d3274ec..8b110b2babfaf5 100644
--- a/transaction-scheduler-bench/src/main.rs
+++ b/transaction-scheduler-bench/src/main.rs
@@ -131,7 +131,7 @@ fn spawn_unified_scheduler(
         let mut runnable_queue = solana_scheduler::TaskQueue::default();
 
         solana_scheduler::ScheduleStage::run(
-            num_execution_threads * 2,
+            num_execution_threads * 10,
             &mut runnable_queue,
             &mut address_book,
             &packet_batch_receiver.clone(),

From cf67720ed8bb1c2e350f4023fc8512144f8c0dfe Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Wed, 24 Aug 2022 22:11:27 +0900
Subject: [PATCH 1305/3199] save

---
 transaction-scheduler-bench/src/main.rs | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/transaction-scheduler-bench/src/main.rs b/transaction-scheduler-bench/src/main.rs
index 8b110b2babfaf5..f910c18a9faca4 100644
--- a/transaction-scheduler-bench/src/main.rs
+++ b/transaction-scheduler-bench/src/main.rs
@@ -354,7 +354,7 @@ fn handle_transaction_batch(
         .fetch_add(priority_collected, Ordering::Relaxed);
 
     let uq = transaction_batch.unique_weight;
-    transaction_batch.task.trace_timestamps("in_exec(self)");
+    //transaction_batch.task.trace_timestamps("in_exec(self)");
     for mut lock_attempt in transaction_batch.lock_attempts.iter_mut() {
         let contended_unique_weights = lock_attempt.contended_unique_weights();
         contended_unique_weights.remove_task(&uq);
@@ -362,14 +362,14 @@ fn handle_transaction_batch(
             let mut found = true;
             assert_ne!(task_cursor.key(), &uq);
             let mut task = task_cursor.value();
-            task.trace_timestamps("in_exec(initial list)");
+            //task.trace_timestamps("in_exec(initial list)");
             while !task.currently_contended() {
                 if let Some(new_cursor) = task_cursor.prev() {
                     assert!(new_cursor.key() < task_cursor.key());
                     assert_ne!(new_cursor.key(), &uq);
                     task_cursor = new_cursor;
                     task = task_cursor.value();
-                    task.trace_timestamps("in_exec(subsequent list)");
+                    //task.trace_timestamps("in_exec(subsequent list)");
                 } else {
                     found = false;
                     break;
@@ -377,7 +377,7 @@ fn handle_transaction_batch(
             }
             found.then(|| solana_scheduler::TaskInQueue::clone(task))
         }).flatten().map(|task| {
-            task.trace_timestamps("in_exec(heaviest)");
+            //task.trace_timestamps("in_exec(heaviest)");
             lock_attempt.heaviest_uncontended = Some(task);
             ()
         });

From 23b7f4d9fb609692f5f865769f2d99b58afede2d Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Wed, 24 Aug 2022 22:15:37 +0900
Subject: [PATCH 1306/3199] save

---
 transaction-scheduler-bench/src/main.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/transaction-scheduler-bench/src/main.rs b/transaction-scheduler-bench/src/main.rs
index f910c18a9faca4..2f782a7b8f92bc 100644
--- a/transaction-scheduler-bench/src/main.rs
+++ b/transaction-scheduler-bench/src/main.rs
@@ -377,7 +377,7 @@ fn handle_transaction_batch(
             }
             found.then(|| solana_scheduler::TaskInQueue::clone(task))
         }).flatten().map(|task| {
-            //task.trace_timestamps("in_exec(heaviest)");
+            task.trace_timestamps("in_exec(heaviest)");
             lock_attempt.heaviest_uncontended = Some(task);
             ()
         });

From f2142b29296be9eb84811b39e2aadb9377f20954 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Wed, 24 Aug 2022 22:24:36 +0900
Subject: [PATCH 1307/3199] save

---
 transaction-scheduler-bench/src/main.rs | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

diff --git a/transaction-scheduler-bench/src/main.rs b/transaction-scheduler-bench/src/main.rs
index 2f782a7b8f92bc..759829359d8c47 100644
--- a/transaction-scheduler-bench/src/main.rs
+++ b/transaction-scheduler-bench/src/main.rs
@@ -357,16 +357,19 @@ fn handle_transaction_batch(
     //transaction_batch.task.trace_timestamps("in_exec(self)");
     for mut lock_attempt in transaction_batch.lock_attempts.iter_mut() {
         let contended_unique_weights = lock_attempt.contended_unique_weights();
-        contended_unique_weights.remove_task(&uq);
         contended_unique_weights.heaviest_task_cursor().map(|mut task_cursor| {
             let mut found = true;
+            let mut removed = false;
             assert_ne!(task_cursor.key(), &uq);
             let mut task = task_cursor.value();
             //task.trace_timestamps("in_exec(initial list)");
             while !task.currently_contended() {
+                if task_cursor.key() == &uq {
+                    task_cursor.remove();
+                    removed = true;
+                }
                 if let Some(new_cursor) = task_cursor.prev() {
                     assert!(new_cursor.key() < task_cursor.key());
-                    assert_ne!(new_cursor.key(), &uq);
                     task_cursor = new_cursor;
                     task = task_cursor.value();
                     //task.trace_timestamps("in_exec(subsequent list)");
@@ -375,6 +378,9 @@ fn handle_transaction_batch(
                     break;
                 }
             }
+            if !removed {
+                contended_unique_weights.remove_task(&uq);
+            }
             found.then(|| solana_scheduler::TaskInQueue::clone(task))
         }).flatten().map(|task| {
             task.trace_timestamps("in_exec(heaviest)");

From 68fbd12b52f4101ef9b7d26e39e0b98a1db3fac0 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Wed, 24 Aug 2022 22:25:45 +0900
Subject: [PATCH 1308/3199] save

---
 transaction-scheduler-bench/src/main.rs | 1 -
 1 file changed, 1 deletion(-)

diff --git a/transaction-scheduler-bench/src/main.rs b/transaction-scheduler-bench/src/main.rs
index 759829359d8c47..0d99a9cee93d6c 100644
--- a/transaction-scheduler-bench/src/main.rs
+++ b/transaction-scheduler-bench/src/main.rs
@@ -360,7 +360,6 @@ fn handle_transaction_batch(
         contended_unique_weights.heaviest_task_cursor().map(|mut task_cursor| {
             let mut found = true;
             let mut removed = false;
-            assert_ne!(task_cursor.key(), &uq);
             let mut task = task_cursor.value();
             //task.trace_timestamps("in_exec(initial list)");
             while !task.currently_contended() {

From b4e869fd2a273a35349d15a7965ab85c5b358d39 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Wed, 24 Aug 2022 22:26:17 +0900
Subject: [PATCH 1309/3199] save

---
 transaction-scheduler-bench/src/main.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/transaction-scheduler-bench/src/main.rs b/transaction-scheduler-bench/src/main.rs
index 0d99a9cee93d6c..edc4f84e76d678 100644
--- a/transaction-scheduler-bench/src/main.rs
+++ b/transaction-scheduler-bench/src/main.rs
@@ -364,7 +364,7 @@ fn handle_transaction_batch(
             //task.trace_timestamps("in_exec(initial list)");
             while !task.currently_contended() {
                 if task_cursor.key() == &uq {
-                    task_cursor.remove();
+                    assert!(task_cursor.remove());
                     removed = true;
                 }
                 if let Some(new_cursor) = task_cursor.prev() {

From e208134caf0bd193218e99530b857fdab1c31ca9 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Wed, 24 Aug 2022 22:28:54 +0900
Subject: [PATCH 1310/3199] save

---
 transaction-scheduler-bench/src/main.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/transaction-scheduler-bench/src/main.rs b/transaction-scheduler-bench/src/main.rs
index edc4f84e76d678..af3ea2a26abf11 100644
--- a/transaction-scheduler-bench/src/main.rs
+++ b/transaction-scheduler-bench/src/main.rs
@@ -354,7 +354,7 @@ fn handle_transaction_batch(
         .fetch_add(priority_collected, Ordering::Relaxed);
 
     let uq = transaction_batch.unique_weight;
-    //transaction_batch.task.trace_timestamps("in_exec(self)");
+    transaction_batch.task.trace_timestamps("in_exec(self)");
     for mut lock_attempt in transaction_batch.lock_attempts.iter_mut() {
         let contended_unique_weights = lock_attempt.contended_unique_weights();
         contended_unique_weights.heaviest_task_cursor().map(|mut task_cursor| {

From 49bbe6307758f9e3fc33d3774618806ffb352aa4 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Wed, 24 Aug 2022 22:31:48 +0900
Subject: [PATCH 1311/3199] save

---
 transaction-scheduler-bench/src/main.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/transaction-scheduler-bench/src/main.rs b/transaction-scheduler-bench/src/main.rs
index af3ea2a26abf11..4a4490820cb071 100644
--- a/transaction-scheduler-bench/src/main.rs
+++ b/transaction-scheduler-bench/src/main.rs
@@ -354,7 +354,7 @@ fn handle_transaction_batch(
         .fetch_add(priority_collected, Ordering::Relaxed);
 
     let uq = transaction_batch.unique_weight;
-    transaction_batch.task.trace_timestamps("in_exec(self)");
+    //transaction_batch.task.trace_timestamps("in_exec(self)");
     for mut lock_attempt in transaction_batch.lock_attempts.iter_mut() {
         let contended_unique_weights = lock_attempt.contended_unique_weights();
         contended_unique_weights.heaviest_task_cursor().map(|mut task_cursor| {
@@ -382,7 +382,7 @@ fn handle_transaction_batch(
             }
             found.then(|| solana_scheduler::TaskInQueue::clone(task))
         }).flatten().map(|task| {
-            task.trace_timestamps("in_exec(heaviest)");
+            task.trace_timestamps(&format!("in_exec(heaviest)"));
             lock_attempt.heaviest_uncontended = Some(task);
             ()
         });

From bdb71818e77cbc17978ea62dd486179d78d1779b Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Wed, 24 Aug 2022 22:34:02 +0900
Subject: [PATCH 1312/3199] save

---
 scheduler/src/lib.rs                    | 6 ++++++
 transaction-scheduler-bench/src/main.rs | 2 +-
 2 files changed, 7 insertions(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index bdc9ff8eefc0be..bda2959fc519bd 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -485,6 +485,12 @@ impl Task {
         self.commit_time.load(std::sync::atomic::Ordering::SeqCst)
     }
 
+    pub fn queue_time_label(&self) -> &str {
+        format!("queue: [{}qT..{}qT; {}qD]", 
+              self.queue_time(), self.queue_end_time(), self.queue_end_time() - self.queue_time(), 
+              )
+    }
+
     pub fn trace_timestamps(&self, prefix: &str) {
         trace!("{}: {:016x} seq: [{}sT..{}sT; {}sD], queue: [{}qT..{}qT; {}qD] exec: [{}eT..{}eT; {}eD]", 
               prefix,
diff --git a/transaction-scheduler-bench/src/main.rs b/transaction-scheduler-bench/src/main.rs
index 4a4490820cb071..0631dc70eecfd2 100644
--- a/transaction-scheduler-bench/src/main.rs
+++ b/transaction-scheduler-bench/src/main.rs
@@ -382,7 +382,7 @@ fn handle_transaction_batch(
             }
             found.then(|| solana_scheduler::TaskInQueue::clone(task))
         }).flatten().map(|task| {
-            task.trace_timestamps(&format!("in_exec(heaviest)"));
+            task.trace_timestamps(&format!("in_exec(heaviest:{})", transaction abatch.task.queue_time_label()));
             lock_attempt.heaviest_uncontended = Some(task);
             ()
         });

From d02aa2c73c2ba07d8f749398e5550b8d657fc8ed Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Wed, 24 Aug 2022 22:34:17 +0900
Subject: [PATCH 1313/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index bda2959fc519bd..f1c4cf88d57462 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -486,7 +486,7 @@ impl Task {
     }
 
     pub fn queue_time_label(&self) -> &str {
-        format!("queue: [{}qT..{}qT; {}qD]", 
+        &format!("queue: [{}qT..{}qT; {}qD]", 
               self.queue_time(), self.queue_end_time(), self.queue_end_time() - self.queue_time(), 
               )
     }

From 7962318d818bed050a08f65864e2901e10e04617 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Wed, 24 Aug 2022 22:34:30 +0900
Subject: [PATCH 1314/3199] save

---
 scheduler/src/lib.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index f1c4cf88d57462..e47cfe4a9ba058 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -485,8 +485,8 @@ impl Task {
         self.commit_time.load(std::sync::atomic::Ordering::SeqCst)
     }
 
-    pub fn queue_time_label(&self) -> &str {
-        &format!("queue: [{}qT..{}qT; {}qD]", 
+    pub fn queue_time_label(&self) -> String {
+        format!("queue: [{}qT..{}qT; {}qD]", 
               self.queue_time(), self.queue_end_time(), self.queue_end_time() - self.queue_time(), 
               )
     }

From 3fe24c5f07a7de40a6509cc04f3a3348c10c5f1a Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Wed, 24 Aug 2022 22:34:42 +0900
Subject: [PATCH 1315/3199] save

---
 transaction-scheduler-bench/src/main.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/transaction-scheduler-bench/src/main.rs b/transaction-scheduler-bench/src/main.rs
index 0631dc70eecfd2..79cb7035bc5e9e 100644
--- a/transaction-scheduler-bench/src/main.rs
+++ b/transaction-scheduler-bench/src/main.rs
@@ -382,7 +382,7 @@ fn handle_transaction_batch(
             }
             found.then(|| solana_scheduler::TaskInQueue::clone(task))
         }).flatten().map(|task| {
-            task.trace_timestamps(&format!("in_exec(heaviest:{})", transaction abatch.task.queue_time_label()));
+            task.trace_timestamps(&format!("in_exec(heaviest:{})", transaction_batch.task.queue_time_label()));
             lock_attempt.heaviest_uncontended = Some(task);
             ()
         });

From fdd777ed6a20a837d1000e60b1ecae792f97ddb2 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Wed, 24 Aug 2022 22:42:33 +0900
Subject: [PATCH 1316/3199] save

---
 scheduler/src/lib.rs | 18 ++++--------------
 1 file changed, 4 insertions(+), 14 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index e47cfe4a9ba058..cb89292b5f0c63 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -690,31 +690,21 @@ impl ScheduleStage {
             }
             (None, Some(weight_from_contended)) => {
                 trace!("select: contended only");
-                let uw = *weight_from_contended.key();
                 let t = weight_from_contended.remove();
-
                 return Some(( false, t))
             },
             (Some(heaviest_runnable_entry), Some(weight_from_contended)) => {
                 let weight_from_runnable = heaviest_runnable_entry.key();
                 let uw = weight_from_contended.key();
 
-                if true || weight_from_runnable > uw {
+                if weight_from_runnable > uw {
                     trace!("select: runnable > contended");
                     let t = heaviest_runnable_entry.remove();
                     return Some((true, t))
-                } else if false && uw > weight_from_runnable {
-                    panic!();
-                    /*
+                } else if uw > weight_from_runnable {
                     trace!("select: contended > runnnable");
-                    let uw = *uw;
-                    weight_from_contended.remove();
-                    if let Some (queue_entry) = contended_queue.entry_to_execute(uw) {
-                        return Some(( None, queue_entry))
-                    } else {
-                        unreachable!();
-                    }
-                    */
+                    let t = weight_from_contended.remove();
+                    return Some(( false, t))
                 } else {
                     unreachable!(
                         "identical unique weights shouldn't exist in both runnable and contended"

From b535fd5c127b5973957685932cbf00bc1132161b Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Wed, 24 Aug 2022 22:45:23 +0900
Subject: [PATCH 1317/3199] save

---
 scheduler/src/lib.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index cb89292b5f0c63..88108112be3739 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -697,11 +697,11 @@ impl ScheduleStage {
                 let weight_from_runnable = heaviest_runnable_entry.key();
                 let uw = weight_from_contended.key();
 
-                if weight_from_runnable > uw {
+                if false && weight_from_runnable > uw {
                     trace!("select: runnable > contended");
                     let t = heaviest_runnable_entry.remove();
                     return Some((true, t))
-                } else if uw > weight_from_runnable {
+                } else if true || uw > weight_from_runnable {
                     trace!("select: contended > runnnable");
                     let t = weight_from_contended.remove();
                     return Some(( false, t))

From 5653b8f6c3a7813a9e574c2693f4bf590d59b23d Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Wed, 24 Aug 2022 22:47:43 +0900
Subject: [PATCH 1318/3199] save

---
 scheduler/src/lib.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 88108112be3739..316b1b29e9adc1 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -872,7 +872,7 @@ impl ScheduleStage {
                     }
                 }
 
-                /*if !inserted {
+                if !inserted {
                     let contended_unique_weights = &page.contended_unique_weights;
                     contended_unique_weights.heaviest_task_cursor().map(|mut task_cursor| {
                         let mut found = true;
@@ -894,7 +894,7 @@ impl ScheduleStage {
                         address_book.uncontended_task_ids.insert(task.unique_weight, task);
                         ()
                     });
-                }*/
+                }
             }
             if page.current_usage == Usage::Unused && page.next_usage != Usage::Unused {
                 page.switch_to_next_usage();

From 90b2dcc1000dd63d2e556e6e26bb8a365f0e9fca Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Wed, 24 Aug 2022 22:49:03 +0900
Subject: [PATCH 1319/3199] save

---
 scheduler/src/lib.rs | 46 +++++++++++++++++++++-----------------------
 1 file changed, 22 insertions(+), 24 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 316b1b29e9adc1..016e7627157074 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -869,31 +869,29 @@ impl ScheduleStage {
                     if task.currently_contended() {
                         inserted = true;
                         address_book.uncontended_task_ids.insert(task.unique_weight, task);
-                    }
-                }
-
-                if !inserted {
-                    let contended_unique_weights = &page.contended_unique_weights;
-                    contended_unique_weights.heaviest_task_cursor().map(|mut task_cursor| {
-                        let mut found = true;
-                        //assert_ne!(task_cursor.key(), &task.uq);
-                        let mut task = task_cursor.value();
-                        while !task.currently_contended() {
-                            if let Some(new_cursor) = task_cursor.prev() {
-                                assert!(new_cursor.key() < task_cursor.key());
-                                //assert_ne!(new_cursor.key(), &uq);
-                                task_cursor = new_cursor;
-                                task = task_cursor.value();
-                            } else {
-                                found = false;
-                                break;
+                    } else {
+                        let contended_unique_weights = &page.contended_unique_weights;
+                        contended_unique_weights.heaviest_task_cursor().map(|mut task_cursor| {
+                            let mut found = true;
+                            //assert_ne!(task_cursor.key(), &task.uq);
+                            let mut task = task_cursor.value();
+                            while !task.currently_contended() {
+                                if let Some(new_cursor) = task_cursor.prev() {
+                                    assert!(new_cursor.key() < task_cursor.key());
+                                    //assert_ne!(new_cursor.key(), &uq);
+                                    task_cursor = new_cursor;
+                                    task = task_cursor.value();
+                                } else {
+                                    found = false;
+                                    break;
+                                }
                             }
-                        }
-                        found.then(|| TaskInQueue::clone(task))
-                    }).flatten().map(|task| {
-                        address_book.uncontended_task_ids.insert(task.unique_weight, task);
-                        ()
-                    });
+                            found.then(|| TaskInQueue::clone(task))
+                        }).flatten().map(|task| {
+                            address_book.uncontended_task_ids.insert(task.unique_weight, task);
+                            ()
+                        });
+                    }
                 }
             }
             if page.current_usage == Usage::Unused && page.next_usage != Usage::Unused {

From 0a7a3be452abc5d1cdbad1f5bb4999bbf5b68b56 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Wed, 24 Aug 2022 22:49:47 +0900
Subject: [PATCH 1320/3199] save

---
 scheduler/src/lib.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 016e7627157074..a302300c4b6e3e 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -869,7 +869,7 @@ impl ScheduleStage {
                     if task.currently_contended() {
                         inserted = true;
                         address_book.uncontended_task_ids.insert(task.unique_weight, task);
-                    } else {
+                    } /*else {
                         let contended_unique_weights = &page.contended_unique_weights;
                         contended_unique_weights.heaviest_task_cursor().map(|mut task_cursor| {
                             let mut found = true;
@@ -891,7 +891,7 @@ impl ScheduleStage {
                             address_book.uncontended_task_ids.insert(task.unique_weight, task);
                             ()
                         });
-                    }
+                    }*/
                 }
             }
             if page.current_usage == Usage::Unused && page.next_usage != Usage::Unused {

From 4e7ffe00dfd72d355cbd948361e96c7042839b49 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Wed, 24 Aug 2022 22:51:30 +0900
Subject: [PATCH 1321/3199] save

---
 scheduler/src/lib.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index a302300c4b6e3e..a8eebc350a47a9 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -697,11 +697,11 @@ impl ScheduleStage {
                 let weight_from_runnable = heaviest_runnable_entry.key();
                 let uw = weight_from_contended.key();
 
-                if false && weight_from_runnable > uw {
+                if weight_from_runnable > uw {
                     trace!("select: runnable > contended");
                     let t = heaviest_runnable_entry.remove();
                     return Some((true, t))
-                } else if true || uw > weight_from_runnable {
+                } else if uw > weight_from_runnable {
                     trace!("select: contended > runnnable");
                     let t = weight_from_contended.remove();
                     return Some(( false, t))

From 77f9321ad352c67ac47646577775b8872b1c0c94 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Wed, 24 Aug 2022 22:52:09 +0900
Subject: [PATCH 1322/3199] save

---
 scheduler/src/lib.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index a8eebc350a47a9..ec64b3ed0bd6b8 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -697,11 +697,11 @@ impl ScheduleStage {
                 let weight_from_runnable = heaviest_runnable_entry.key();
                 let uw = weight_from_contended.key();
 
-                if weight_from_runnable > uw {
+                if true || weight_from_runnable > uw {
                     trace!("select: runnable > contended");
                     let t = heaviest_runnable_entry.remove();
                     return Some((true, t))
-                } else if uw > weight_from_runnable {
+                } else if false && uw > weight_from_runnable {
                     trace!("select: contended > runnnable");
                     let t = weight_from_contended.remove();
                     return Some(( false, t))

From e3c734a48583ee4cb2e704da6d05197495e97766 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 25 Aug 2022 15:58:40 +0900
Subject: [PATCH 1323/3199] save

---
 transaction-scheduler-bench/src/main.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/transaction-scheduler-bench/src/main.rs b/transaction-scheduler-bench/src/main.rs
index 79cb7035bc5e9e..5ed022e3cdd2a9 100644
--- a/transaction-scheduler-bench/src/main.rs
+++ b/transaction-scheduler-bench/src/main.rs
@@ -364,8 +364,8 @@ fn handle_transaction_batch(
             //task.trace_timestamps("in_exec(initial list)");
             while !task.currently_contended() {
                 if task_cursor.key() == &uq {
-                    assert!(task_cursor.remove());
-                    removed = true;
+                    removed = task_cursor.remove();
+                    assert!(removed);
                 }
                 if let Some(new_cursor) = task_cursor.prev() {
                     assert!(new_cursor.key() < task_cursor.key());

From 5582ff7dfa75f70d54bf38184e47f9a3cb33807c Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 25 Aug 2022 16:16:29 +0900
Subject: [PATCH 1324/3199] save

---
 scheduler/src/lib.rs | 1 +
 1 file changed, 1 insertion(+)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index ec64b3ed0bd6b8..1b72bd493cc24e 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -788,6 +788,7 @@ impl ScheduleStage {
                 }
             } else if provisional_count > 0 {
                 assert!(!from_runnable);
+                assert_eq!(unlockable_count, 0);
                 let lock_count = next_task.tx.1.len();
                 trace!("provisional exec: [{}/{}]", provisional_count, lock_count);
                 Self::finalize_lock_for_provisional_execution(

From 7b81781e9354a923ffb6ae21c40c306ae4962c0a Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 25 Aug 2022 16:20:15 +0900
Subject: [PATCH 1325/3199] save

---
 scheduler/src/lib.rs | 1 +
 1 file changed, 1 insertion(+)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 1b72bd493cc24e..bd42033e35c20d 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1113,6 +1113,7 @@ impl ScheduleStage {
                     executing_queue_count += 1;
 
                     to_execute_substage.send(ee).unwrap();
+                    break;
                 } else {
                     trace!("incoming queue starved");
                     while from.len() > 0 {

From 6362f6eb03507e8b67331d1a40e9cc7ae420b79f Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 25 Aug 2022 16:46:38 +0900
Subject: [PATCH 1326/3199] save

---
 scheduler/src/lib.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index bd42033e35c20d..51d04023322f33 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -697,11 +697,11 @@ impl ScheduleStage {
                 let weight_from_runnable = heaviest_runnable_entry.key();
                 let uw = weight_from_contended.key();
 
-                if true || weight_from_runnable > uw {
+                if weight_from_runnable > uw {
                     trace!("select: runnable > contended");
                     let t = heaviest_runnable_entry.remove();
                     return Some((true, t))
-                } else if false && uw > weight_from_runnable {
+                } else if uw > weight_from_runnable {
                     trace!("select: contended > runnnable");
                     let t = weight_from_contended.remove();
                     return Some(( false, t))

From 097122ff1b145bb9961a8d0cfd22273faada4791 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 25 Aug 2022 16:58:36 +0900
Subject: [PATCH 1327/3199] save

---
 transaction-scheduler-bench/src/main.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/transaction-scheduler-bench/src/main.rs b/transaction-scheduler-bench/src/main.rs
index 5ed022e3cdd2a9..bee44dc36117e4 100644
--- a/transaction-scheduler-bench/src/main.rs
+++ b/transaction-scheduler-bench/src/main.rs
@@ -131,7 +131,7 @@ fn spawn_unified_scheduler(
         let mut runnable_queue = solana_scheduler::TaskQueue::default();
 
         solana_scheduler::ScheduleStage::run(
-            num_execution_threads * 10,
+            num_execution_threads * 2,
             &mut runnable_queue,
             &mut address_book,
             &packet_batch_receiver.clone(),

From 22f4ea5678d644a35a2874933b937b9a40000daa Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 25 Aug 2022 21:36:21 +0900
Subject: [PATCH 1328/3199] save

---
 transaction-scheduler-bench/src/main.rs | 24 +++++++++++++-----------
 1 file changed, 13 insertions(+), 11 deletions(-)

diff --git a/transaction-scheduler-bench/src/main.rs b/transaction-scheduler-bench/src/main.rs
index bee44dc36117e4..25110c16957c37 100644
--- a/transaction-scheduler-bench/src/main.rs
+++ b/transaction-scheduler-bench/src/main.rs
@@ -491,20 +491,22 @@ fn send_packets(
         );
         let mut rng = rand::thread_rng();
 
-        for vv in packet_batches {
-            for v in vv {
-                let p = solana_scheduler::get_transaction_priority_details(&v.0);
-                let p = (p << 32) | (rng.gen::<u64>() & 0x0000_0000_ffff_ffff);
-                let t = solana_scheduler::Task::new_for_queue(p, v);
-                for lock_attempt in t.tx.1.iter() {
-                    lock_attempt.contended_unique_weights().insert_task(p, solana_scheduler::TaskInQueue::clone(&t));
+        loop {
+            for vv in packet_batches {
+                for v in vv {
+                    let p = solana_scheduler::get_transaction_priority_details(&v.0);
+                    let p = (p << 32) | (rng.gen::<u64>() & 0x0000_0000_ffff_ffff);
+                    let t = solana_scheduler::Task::new_for_queue(p, v);
+                    for lock_attempt in t.tx.1.iter() {
+                        lock_attempt.contended_unique_weights().insert_task(p, solana_scheduler::TaskInQueue::clone(&t));
+                    }
+                    packet_batch_sender.send(solana_scheduler::Multiplexed::FromPrevious((p, t))).unwrap();
                 }
-                packet_batch_sender.send(solana_scheduler::Multiplexed::FromPrevious((p, t))).unwrap();
             }
-        }
-        
+            
 
-        std::thread::sleep(loop_duration.saturating_sub(packet_build_time.as_duration()));
+            std::thread::sleep(loop_duration.saturating_sub(packet_build_time.as_duration()));
+        }
     }
 }
 

From 90ee28227bb34ab1dc528420450b9ace540721aa Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 25 Aug 2022 21:36:37 +0900
Subject: [PATCH 1329/3199] save

---
 transaction-scheduler-bench/src/main.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/transaction-scheduler-bench/src/main.rs b/transaction-scheduler-bench/src/main.rs
index 25110c16957c37..c22dfbc972d02b 100644
--- a/transaction-scheduler-bench/src/main.rs
+++ b/transaction-scheduler-bench/src/main.rs
@@ -492,7 +492,7 @@ fn send_packets(
         let mut rng = rand::thread_rng();
 
         loop {
-            for vv in packet_batches {
+            for vv in packet_batches.clone_for_test() {
                 for v in vv {
                     let p = solana_scheduler::get_transaction_priority_details(&v.0);
                     let p = (p << 32) | (rng.gen::<u64>() & 0x0000_0000_ffff_ffff);

From 25ee77638ae1861f071aa7e9199c4db65b639539 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 25 Aug 2022 21:36:45 +0900
Subject: [PATCH 1330/3199] save

---
 transaction-scheduler-bench/src/main.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/transaction-scheduler-bench/src/main.rs b/transaction-scheduler-bench/src/main.rs
index c22dfbc972d02b..80ef344dcdeca4 100644
--- a/transaction-scheduler-bench/src/main.rs
+++ b/transaction-scheduler-bench/src/main.rs
@@ -492,8 +492,8 @@ fn send_packets(
         let mut rng = rand::thread_rng();
 
         loop {
-            for vv in packet_batches.clone_for_test() {
-                for v in vv {
+            for vv in packet_batches {
+                for v in vv.clone_for_test() {
                     let p = solana_scheduler::get_transaction_priority_details(&v.0);
                     let p = (p << 32) | (rng.gen::<u64>() & 0x0000_0000_ffff_ffff);
                     let t = solana_scheduler::Task::new_for_queue(p, v);

From 13a66d3f91591e78b238e3805a5ac55e91a295cc Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 25 Aug 2022 21:37:55 +0900
Subject: [PATCH 1331/3199] save

---
 transaction-scheduler-bench/src/main.rs | 26 ++++++++++++-------------
 1 file changed, 12 insertions(+), 14 deletions(-)

diff --git a/transaction-scheduler-bench/src/main.rs b/transaction-scheduler-bench/src/main.rs
index 80ef344dcdeca4..5ed022e3cdd2a9 100644
--- a/transaction-scheduler-bench/src/main.rs
+++ b/transaction-scheduler-bench/src/main.rs
@@ -131,7 +131,7 @@ fn spawn_unified_scheduler(
         let mut runnable_queue = solana_scheduler::TaskQueue::default();
 
         solana_scheduler::ScheduleStage::run(
-            num_execution_threads * 2,
+            num_execution_threads * 10,
             &mut runnable_queue,
             &mut address_book,
             &packet_batch_receiver.clone(),
@@ -491,22 +491,20 @@ fn send_packets(
         );
         let mut rng = rand::thread_rng();
 
-        loop {
-            for vv in packet_batches {
-                for v in vv.clone_for_test() {
-                    let p = solana_scheduler::get_transaction_priority_details(&v.0);
-                    let p = (p << 32) | (rng.gen::<u64>() & 0x0000_0000_ffff_ffff);
-                    let t = solana_scheduler::Task::new_for_queue(p, v);
-                    for lock_attempt in t.tx.1.iter() {
-                        lock_attempt.contended_unique_weights().insert_task(p, solana_scheduler::TaskInQueue::clone(&t));
-                    }
-                    packet_batch_sender.send(solana_scheduler::Multiplexed::FromPrevious((p, t))).unwrap();
+        for vv in packet_batches {
+            for v in vv {
+                let p = solana_scheduler::get_transaction_priority_details(&v.0);
+                let p = (p << 32) | (rng.gen::<u64>() & 0x0000_0000_ffff_ffff);
+                let t = solana_scheduler::Task::new_for_queue(p, v);
+                for lock_attempt in t.tx.1.iter() {
+                    lock_attempt.contended_unique_weights().insert_task(p, solana_scheduler::TaskInQueue::clone(&t));
                 }
+                packet_batch_sender.send(solana_scheduler::Multiplexed::FromPrevious((p, t))).unwrap();
             }
-            
-
-            std::thread::sleep(loop_duration.saturating_sub(packet_build_time.as_duration()));
         }
+        
+
+        std::thread::sleep(loop_duration.saturating_sub(packet_build_time.as_duration()));
     }
 }
 

From 2edb1f978dd6401faa463d446417e37c9bc847bb Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 25 Aug 2022 21:42:38 +0900
Subject: [PATCH 1332/3199] save

---
 transaction-scheduler-bench/src/main.rs | 14 ++++++++++----
 1 file changed, 10 insertions(+), 4 deletions(-)

diff --git a/transaction-scheduler-bench/src/main.rs b/transaction-scheduler-bench/src/main.rs
index 5ed022e3cdd2a9..3ee88261326f03 100644
--- a/transaction-scheduler-bench/src/main.rs
+++ b/transaction-scheduler-bench/src/main.rs
@@ -393,8 +393,6 @@ fn handle_transaction_batch(
         .unwrap();
 }
 
-const NUM_SENDERS: usize = 2;
-
 fn spawn_packet_senders(
     preloader: Arc<solana_scheduler::Preloader>,
     metrics: Arc<TransactionSchedulerBenchMetrics>,
@@ -405,7 +403,11 @@ fn spawn_packet_senders(
     duration: Duration,
     exit: Arc<AtomicBool>,
 ) -> Vec<JoinHandle<()>> {
-    (0..NUM_SENDERS)
+    let producer_count = std::env::var("PRODUCER_COUNT")
+        .unwrap_or(format!("{}", 4))
+        .parse::<usize>()
+        .unwrap();
+    (0..producer_count)
         .map(|i| {
             let num_accounts = if i == 0 && high_conflict_sender > 0 {
                 high_conflict_sender
@@ -413,6 +415,7 @@ fn spawn_packet_senders(
                 accounts.len()
             };
             spawn_packet_sender(
+                producer_count,
                 Arc::clone(&preloader),
                 metrics.clone(),
                 num_accounts,
@@ -427,6 +430,7 @@ fn spawn_packet_senders(
 }
 
 fn spawn_packet_sender(
+    producer_count: usize,
     preloader: Arc<solana_scheduler::Preloader>,
     metrics: Arc<TransactionSchedulerBenchMetrics>,
     num_accounts: usize,
@@ -438,6 +442,7 @@ fn spawn_packet_sender(
 ) -> JoinHandle<()> {
     std::thread::Builder::new().name("sol-producer".to_string()).spawn(move || {
         send_packets(
+            producer_count,
             preloader,
             metrics,
             num_accounts,
@@ -451,6 +456,7 @@ fn spawn_packet_sender(
 }
 
 fn send_packets(
+    producer_count: usize,
     preloader: Arc<solana_scheduler::Preloader>,
     metrics: Arc<TransactionSchedulerBenchMetrics>,
     num_accounts: usize,
@@ -462,7 +468,7 @@ fn send_packets(
 ) {
     let packets_per_msg = config.packets_per_batch * config.batches_per_msg;
     let loop_frequency =
-        config.packet_send_rate as f64 * packets_per_msg as f64 / NUM_SENDERS as f64;
+        config.packet_send_rate as f64 * packets_per_msg as f64 / producer_count as f64;
     let loop_duration = Duration::from_secs_f64(1.0 / loop_frequency);
 
     info!("sending packets: packets_per_msg: {packets_per_msg} loop_frequency: {loop_frequency} loop_duration: {loop_duration:?}");

From 0d5675725f7a936a01edbeb185bd5d793069e92a Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 25 Aug 2022 21:44:19 +0900
Subject: [PATCH 1333/3199] save

---
 transaction-scheduler-bench/src/main.rs | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/transaction-scheduler-bench/src/main.rs b/transaction-scheduler-bench/src/main.rs
index 3ee88261326f03..41092d6998eb01 100644
--- a/transaction-scheduler-bench/src/main.rs
+++ b/transaction-scheduler-bench/src/main.rs
@@ -415,6 +415,7 @@ fn spawn_packet_senders(
                 accounts.len()
             };
             spawn_packet_sender(
+                i,
                 producer_count,
                 Arc::clone(&preloader),
                 metrics.clone(),
@@ -430,6 +431,7 @@ fn spawn_packet_senders(
 }
 
 fn spawn_packet_sender(
+    i: usize,
     producer_count: usize,
     preloader: Arc<solana_scheduler::Preloader>,
     metrics: Arc<TransactionSchedulerBenchMetrics>,
@@ -440,7 +442,7 @@ fn spawn_packet_sender(
     duration: Duration,
     exit: Arc<AtomicBool>,
 ) -> JoinHandle<()> {
-    std::thread::Builder::new().name("sol-producer".to_string()).spawn(move || {
+    std::thread::Builder::new().name(format!("sol-producer{:02}", i)).spawn(move || {
         send_packets(
             producer_count,
             preloader,

From 983a9a9db428d1bb4aeb76b250518aac15e47d55 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 25 Aug 2022 21:47:44 +0900
Subject: [PATCH 1334/3199] save

---
 transaction-scheduler-bench/src/main.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/transaction-scheduler-bench/src/main.rs b/transaction-scheduler-bench/src/main.rs
index 41092d6998eb01..cea15d1fe055b0 100644
--- a/transaction-scheduler-bench/src/main.rs
+++ b/transaction-scheduler-bench/src/main.rs
@@ -361,7 +361,7 @@ fn handle_transaction_batch(
             let mut found = true;
             let mut removed = false;
             let mut task = task_cursor.value();
-            //task.trace_timestamps("in_exec(initial list)");
+            task.trace_timestamps("in_exec(initial list)");
             while !task.currently_contended() {
                 if task_cursor.key() == &uq {
                     removed = task_cursor.remove();
@@ -371,7 +371,7 @@ fn handle_transaction_batch(
                     assert!(new_cursor.key() < task_cursor.key());
                     task_cursor = new_cursor;
                     task = task_cursor.value();
-                    //task.trace_timestamps("in_exec(subsequent list)");
+                    task.trace_timestamps("in_exec(subsequent list)");
                 } else {
                     found = false;
                     break;

From 2a785e08eaf7a71b052de2b9d2aa890f011315b6 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 25 Aug 2022 21:57:05 +0900
Subject: [PATCH 1335/3199] save

---
 scheduler/src/lib.rs                    | 3 +++
 transaction-scheduler-bench/src/main.rs | 6 +++---
 2 files changed, 6 insertions(+), 3 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 51d04023322f33..16faf99fedf54e 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -775,6 +775,9 @@ impl ScheduleStage {
                     trace!("move to contended due to lock failure [{}/{}/{}]", unlockable_count, provisional_count, lock_count);
                     next_task.mark_as_contended();
                     *contended_count = contended_count.checked_add(1).unwrap();
+                    for lock_attempt in next_task.tx.1.iter() {
+                        lock_attempt.contended_unique_weights().insert_task(p, solana_scheduler::TaskInQueue::clone(&t));
+                    }
                     // maybe run lightweight prune logic on contended_queue here.
                 } else {
                     trace!("relock failed [{}/{}/{}]; remains in contended: {:?} contention: {}", unlockable_count, provisional_count, lock_count, &unique_weight, next_task.contention_count);
diff --git a/transaction-scheduler-bench/src/main.rs b/transaction-scheduler-bench/src/main.rs
index cea15d1fe055b0..f4c472cc2c8d30 100644
--- a/transaction-scheduler-bench/src/main.rs
+++ b/transaction-scheduler-bench/src/main.rs
@@ -504,9 +504,9 @@ fn send_packets(
                 let p = solana_scheduler::get_transaction_priority_details(&v.0);
                 let p = (p << 32) | (rng.gen::<u64>() & 0x0000_0000_ffff_ffff);
                 let t = solana_scheduler::Task::new_for_queue(p, v);
-                for lock_attempt in t.tx.1.iter() {
-                    lock_attempt.contended_unique_weights().insert_task(p, solana_scheduler::TaskInQueue::clone(&t));
-                }
+                //for lock_attempt in t.tx.1.iter() {
+                //    lock_attempt.contended_unique_weights().insert_task(p, solana_scheduler::TaskInQueue::clone(&t));
+                //}
                 packet_batch_sender.send(solana_scheduler::Multiplexed::FromPrevious((p, t))).unwrap();
             }
         }

From 2f6d09ec519cda9697383ba8d7c2063e51704f34 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 25 Aug 2022 21:57:19 +0900
Subject: [PATCH 1336/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 16faf99fedf54e..8258531548bb4f 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -776,7 +776,7 @@ impl ScheduleStage {
                     next_task.mark_as_contended();
                     *contended_count = contended_count.checked_add(1).unwrap();
                     for lock_attempt in next_task.tx.1.iter() {
-                        lock_attempt.contended_unique_weights().insert_task(p, solana_scheduler::TaskInQueue::clone(&t));
+                        lock_attempt.contended_unique_weights().insert_task(p, TaskInQueue::clone(&t));
                     }
                     // maybe run lightweight prune logic on contended_queue here.
                 } else {

From d834e5cd0a00934ad527ed1ef94d436afc08cfa2 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 25 Aug 2022 21:57:33 +0900
Subject: [PATCH 1337/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 8258531548bb4f..5c69df04c11c65 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -776,7 +776,7 @@ impl ScheduleStage {
                     next_task.mark_as_contended();
                     *contended_count = contended_count.checked_add(1).unwrap();
                     for lock_attempt in next_task.tx.1.iter() {
-                        lock_attempt.contended_unique_weights().insert_task(p, TaskInQueue::clone(&t));
+                        lock_attempt.contended_unique_weights().insert_task(p, TaskInQueue::clone(&next_task_arc));
                     }
                     // maybe run lightweight prune logic on contended_queue here.
                 } else {

From 48946eb4b872c6f9c74af5afb2f838529919bbff Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 25 Aug 2022 21:57:40 +0900
Subject: [PATCH 1338/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 5c69df04c11c65..29d6c81d62fa11 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -776,7 +776,7 @@ impl ScheduleStage {
                     next_task.mark_as_contended();
                     *contended_count = contended_count.checked_add(1).unwrap();
                     for lock_attempt in next_task.tx.1.iter() {
-                        lock_attempt.contended_unique_weights().insert_task(p, TaskInQueue::clone(&next_task_arc));
+                        lock_attempt.contended_unique_weights().insert_task(p, TaskInQueue::clone(&next_task));
                     }
                     // maybe run lightweight prune logic on contended_queue here.
                 } else {

From 7f8f94f469dd9aae36a8b9fdb220baef00acaba4 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 25 Aug 2022 21:57:53 +0900
Subject: [PATCH 1339/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 29d6c81d62fa11..0e1ea08752cbab 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -776,7 +776,7 @@ impl ScheduleStage {
                     next_task.mark_as_contended();
                     *contended_count = contended_count.checked_add(1).unwrap();
                     for lock_attempt in next_task.tx.1.iter() {
-                        lock_attempt.contended_unique_weights().insert_task(p, TaskInQueue::clone(&next_task));
+                        lock_attempt.contended_unique_weights().insert_task(p, TaskInQueue::clone(&arc_next_task));
                     }
                     // maybe run lightweight prune logic on contended_queue here.
                 } else {

From bf5ab6a4e9dcde09849ac0b369243b08b5e31def Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 25 Aug 2022 21:58:06 +0900
Subject: [PATCH 1340/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 0e1ea08752cbab..b1ad40383666f3 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -776,7 +776,7 @@ impl ScheduleStage {
                     next_task.mark_as_contended();
                     *contended_count = contended_count.checked_add(1).unwrap();
                     for lock_attempt in next_task.tx.1.iter() {
-                        lock_attempt.contended_unique_weights().insert_task(p, TaskInQueue::clone(&arc_next_task));
+                        lock_attempt.contended_unique_weights().insert_task(unique_weight, TaskInQueue::clone(&arc_next_task));
                     }
                     // maybe run lightweight prune logic on contended_queue here.
                 } else {

From 92f2dd7cdba948569df1f81d5ea6e620858880bb Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 25 Aug 2022 21:59:21 +0900
Subject: [PATCH 1341/3199] save

---
 scheduler/src/lib.rs | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index b1ad40383666f3..dc15de3e0c1379 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -744,6 +744,7 @@ impl ScheduleStage {
                 arc_next_task.record_queue_time(*sequence_clock, *queue_clock);
                 *queue_clock = queue_clock.checked_add(1).unwrap();
             }
+            let a2 = TaskInQueue::clone(arc_next_task);
             let next_task = unsafe { TaskInQueue::get_mut_unchecked(&mut arc_next_task) };
             let unique_weight = next_task.unique_weight;
             let message_hash = next_task.tx.0.message_hash();
@@ -776,7 +777,7 @@ impl ScheduleStage {
                     next_task.mark_as_contended();
                     *contended_count = contended_count.checked_add(1).unwrap();
                     for lock_attempt in next_task.tx.1.iter() {
-                        lock_attempt.contended_unique_weights().insert_task(unique_weight, TaskInQueue::clone(&arc_next_task));
+                        lock_attempt.contended_unique_weights().insert_task(unique_weight, TaskInQueue::clone(&a2));
                     }
                     // maybe run lightweight prune logic on contended_queue here.
                 } else {

From 2196a8d4ee97516aeff726e95a3de6f2d441c78c Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 25 Aug 2022 21:59:33 +0900
Subject: [PATCH 1342/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index dc15de3e0c1379..78bb77853c07c1 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -744,7 +744,7 @@ impl ScheduleStage {
                 arc_next_task.record_queue_time(*sequence_clock, *queue_clock);
                 *queue_clock = queue_clock.checked_add(1).unwrap();
             }
-            let a2 = TaskInQueue::clone(arc_next_task);
+            let a2 = TaskInQueue::clone(&arc_next_task);
             let next_task = unsafe { TaskInQueue::get_mut_unchecked(&mut arc_next_task) };
             let unique_weight = next_task.unique_weight;
             let message_hash = next_task.tx.0.message_hash();

From 8c522d0ee4589196cc9f67a2c83953aee11a0931 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 25 Aug 2022 22:01:46 +0900
Subject: [PATCH 1343/3199] save

---
 scheduler/src/lib.rs | 25 -------------------------
 1 file changed, 25 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 78bb77853c07c1..caef4923da8061 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1041,19 +1041,6 @@ impl ScheduleStage {
 
                             Self::register_runnable_task(weighted_tx, runnable_queue, &mut current_unique_key, &mut sequence_time);
 
-                            while from.len() > 0 && from_exec.len() == 0 {
-                               let i = from.recv().unwrap();
-                                match i {
-                                    Multiplexed::FromPrevious(weighted_tx) => {
-                                        trace!("recv from previous (after starvation)");
-                                        Self::register_runnable_task(weighted_tx, runnable_queue, &mut current_unique_key, &mut sequence_time);
-                                    }
-                                    Multiplexed::FromPreviousBatched(vvv) => {
-                                        unreachable!();
-                                    }
-                                }
-                            }
-
                             while false && from_exec.len() > 0 {
                                 let mut processed_execution_environment = from_exec.recv().unwrap();
                                 trace!("recv from execute: {:?}", processed_execution_environment.unique_weight);
@@ -1093,18 +1080,6 @@ impl ScheduleStage {
                     trace!("prefer emptying n_u_a");
                 } else */ if (executing_queue_count + address_book.provisioning_trackers.len()) >= max_executing_queue_count {
                     trace!("skip scheduling; outgoing queue full");
-                    while from.len() > 0 {
-                       let i = from.recv().unwrap();
-                        match i {
-                            Multiplexed::FromPrevious(weighted_tx) => {
-                                trace!("recv from previous (after starvation)");
-                                Self::register_runnable_task(weighted_tx, runnable_queue, &mut current_unique_key, &mut sequence_time);
-                            }
-                            Multiplexed::FromPreviousBatched(vvv) => {
-                                unreachable!();
-                            }
-                        }
-                    }
                     break;
                 }
 

From 98f806eb137f36c075bc7cc3f670927276ef350d Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 25 Aug 2022 22:03:05 +0900
Subject: [PATCH 1344/3199] save

---
 scheduler/src/lib.rs | 13 -------------
 1 file changed, 13 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index caef4923da8061..4c03a07e20782c 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1094,19 +1094,6 @@ impl ScheduleStage {
                     to_execute_substage.send(ee).unwrap();
                     break;
                 } else {
-                    trace!("incoming queue starved");
-                    while from.len() > 0 {
-                       let i = from.recv().unwrap();
-                        match i {
-                            Multiplexed::FromPrevious(weighted_tx) => {
-                                trace!("recv from previous (after starvation)");
-                                Self::register_runnable_task(weighted_tx, runnable_queue, &mut current_unique_key, &mut sequence_time);
-                            }
-                            Multiplexed::FromPreviousBatched(vvv) => {
-                                unreachable!();
-                            }
-                        }
-                    }
                     break;
                 }
             }

From f2c8912fc5a575f2c326ef80ca3adda4a5699dfe Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 25 Aug 2022 22:06:15 +0900
Subject: [PATCH 1345/3199] save

---
 transaction-scheduler-bench/src/main.rs | 60 +++++++++++++------------
 1 file changed, 31 insertions(+), 29 deletions(-)

diff --git a/transaction-scheduler-bench/src/main.rs b/transaction-scheduler-bench/src/main.rs
index f4c472cc2c8d30..2b291445ff8b46 100644
--- a/transaction-scheduler-bench/src/main.rs
+++ b/transaction-scheduler-bench/src/main.rs
@@ -355,37 +355,39 @@ fn handle_transaction_batch(
 
     let uq = transaction_batch.unique_weight;
     //transaction_batch.task.trace_timestamps("in_exec(self)");
-    for mut lock_attempt in transaction_batch.lock_attempts.iter_mut() {
-        let contended_unique_weights = lock_attempt.contended_unique_weights();
-        contended_unique_weights.heaviest_task_cursor().map(|mut task_cursor| {
-            let mut found = true;
-            let mut removed = false;
-            let mut task = task_cursor.value();
-            task.trace_timestamps("in_exec(initial list)");
-            while !task.currently_contended() {
-                if task_cursor.key() == &uq {
-                    removed = task_cursor.remove();
-                    assert!(removed);
+    if transaction_batch.contention_count > 0 {
+        for mut lock_attempt in transaction_batch.lock_attempts.iter_mut() {
+            let contended_unique_weights = lock_attempt.contended_unique_weights();
+            contended_unique_weights.heaviest_task_cursor().map(|mut task_cursor| {
+                let mut found = true;
+                let mut removed = false;
+                let mut task = task_cursor.value();
+                task.trace_timestamps("in_exec(initial list)");
+                while !task.currently_contended() {
+                    if task_cursor.key() == &uq {
+                        removed = task_cursor.remove();
+                        assert!(removed);
+                    }
+                    if let Some(new_cursor) = task_cursor.prev() {
+                        assert!(new_cursor.key() < task_cursor.key());
+                        task_cursor = new_cursor;
+                        task = task_cursor.value();
+                        task.trace_timestamps("in_exec(subsequent list)");
+                    } else {
+                        found = false;
+                        break;
+                    }
                 }
-                if let Some(new_cursor) = task_cursor.prev() {
-                    assert!(new_cursor.key() < task_cursor.key());
-                    task_cursor = new_cursor;
-                    task = task_cursor.value();
-                    task.trace_timestamps("in_exec(subsequent list)");
-                } else {
-                    found = false;
-                    break;
+                if !removed {
+                    contended_unique_weights.remove_task(&uq);
                 }
-            }
-            if !removed {
-                contended_unique_weights.remove_task(&uq);
-            }
-            found.then(|| solana_scheduler::TaskInQueue::clone(task))
-        }).flatten().map(|task| {
-            task.trace_timestamps(&format!("in_exec(heaviest:{})", transaction_batch.task.queue_time_label()));
-            lock_attempt.heaviest_uncontended = Some(task);
-            ()
-        });
+                found.then(|| solana_scheduler::TaskInQueue::clone(task))
+            }).flatten().map(|task| {
+                task.trace_timestamps(&format!("in_exec(heaviest:{})", transaction_batch.task.queue_time_label()));
+                lock_attempt.heaviest_uncontended = Some(task);
+                ()
+            });
+        }
     }
     //error!("send from execute substage: {:?} seq: {}", uq, transaction_batch.task.sequence_time());
     completed_transaction_sender.0

From e4bbfad002ede44ce67efb833a06344a64651d1e Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 25 Aug 2022 22:06:30 +0900
Subject: [PATCH 1346/3199] save

---
 transaction-scheduler-bench/src/main.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/transaction-scheduler-bench/src/main.rs b/transaction-scheduler-bench/src/main.rs
index 2b291445ff8b46..9082c2f3c5a25f 100644
--- a/transaction-scheduler-bench/src/main.rs
+++ b/transaction-scheduler-bench/src/main.rs
@@ -355,7 +355,7 @@ fn handle_transaction_batch(
 
     let uq = transaction_batch.unique_weight;
     //transaction_batch.task.trace_timestamps("in_exec(self)");
-    if transaction_batch.contention_count > 0 {
+    if transaction_batch.task.contention_count > 0 {
         for mut lock_attempt in transaction_batch.lock_attempts.iter_mut() {
             let contended_unique_weights = lock_attempt.contended_unique_weights();
             contended_unique_weights.heaviest_task_cursor().map(|mut task_cursor| {

From 688ca0cb191dae5086e60d3c1fffb9b9f8ed7006 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 25 Aug 2022 22:10:52 +0900
Subject: [PATCH 1347/3199] save

---
 transaction-scheduler-bench/src/main.rs | 62 ++++++++++++-------------
 1 file changed, 31 insertions(+), 31 deletions(-)

diff --git a/transaction-scheduler-bench/src/main.rs b/transaction-scheduler-bench/src/main.rs
index 9082c2f3c5a25f..888aac7a999740 100644
--- a/transaction-scheduler-bench/src/main.rs
+++ b/transaction-scheduler-bench/src/main.rs
@@ -355,39 +355,39 @@ fn handle_transaction_batch(
 
     let uq = transaction_batch.unique_weight;
     //transaction_batch.task.trace_timestamps("in_exec(self)");
-    if transaction_batch.task.contention_count > 0 {
-        for mut lock_attempt in transaction_batch.lock_attempts.iter_mut() {
-            let contended_unique_weights = lock_attempt.contended_unique_weights();
-            contended_unique_weights.heaviest_task_cursor().map(|mut task_cursor| {
-                let mut found = true;
-                let mut removed = false;
-                let mut task = task_cursor.value();
-                task.trace_timestamps("in_exec(initial list)");
-                while !task.currently_contended() {
-                    if task_cursor.key() == &uq {
-                        removed = task_cursor.remove();
-                        assert!(removed);
-                    }
-                    if let Some(new_cursor) = task_cursor.prev() {
-                        assert!(new_cursor.key() < task_cursor.key());
-                        task_cursor = new_cursor;
-                        task = task_cursor.value();
-                        task.trace_timestamps("in_exec(subsequent list)");
-                    } else {
-                        found = false;
-                        break;
-                    }
+    let should_remove = transaction_batch.task.contention_count > 0;
+    for mut lock_attempt in transaction_batch.lock_attempts.iter_mut() {
+        let contended_unique_weights = lock_attempt.contended_unique_weights();
+        contended_unique_weights.heaviest_task_cursor().map(|mut task_cursor| {
+            let mut found = true;
+            let mut removed = false;
+            let mut task = task_cursor.value();
+            task.trace_timestamps("in_exec(initial list)");
+            while !task.currently_contended() {
+                if task_cursor.key() == &uq {
+                    assert!(should_remove);
+                    removed = task_cursor.remove();
+                    assert!(removed);
                 }
-                if !removed {
-                    contended_unique_weights.remove_task(&uq);
+                if let Some(new_cursor) = task_cursor.prev() {
+                    assert!(new_cursor.key() < task_cursor.key());
+                    task_cursor = new_cursor;
+                    task = task_cursor.value();
+                    task.trace_timestamps("in_exec(subsequent list)");
+                } else {
+                    found = false;
+                    break;
                 }
-                found.then(|| solana_scheduler::TaskInQueue::clone(task))
-            }).flatten().map(|task| {
-                task.trace_timestamps(&format!("in_exec(heaviest:{})", transaction_batch.task.queue_time_label()));
-                lock_attempt.heaviest_uncontended = Some(task);
-                ()
-            });
-        }
+            }
+            if should_remove && !removed {
+                contended_unique_weights.remove_task(&uq);
+            }
+            found.then(|| solana_scheduler::TaskInQueue::clone(task))
+        }).flatten().map(|task| {
+            task.trace_timestamps(&format!("in_exec(heaviest:{})", transaction_batch.task.queue_time_label()));
+            lock_attempt.heaviest_uncontended = Some(task);
+            ()
+        });
     }
     //error!("send from execute substage: {:?} seq: {}", uq, transaction_batch.task.sequence_time());
     completed_transaction_sender.0

From 57ab8574405a389c565575ac2c90d5ad2202cc06 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 25 Aug 2022 22:22:16 +0900
Subject: [PATCH 1348/3199] save

---
 scheduler/src/lib.rs | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 4c03a07e20782c..6cfb88d02ac1ef 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1028,6 +1028,15 @@ impl ScheduleStage {
 
             (&ee_sender, Some(h))
         };
+        let (task_sender, task_receiver) = crossbeam_channel::unbounded::<TaskInQueue>();
+        let h = std::thread::Builder::new().name("sol-indexer".to_string()).spawn(move || {
+            while let task = task_receiver.recv().unwrap() {
+                let unique_weight = task.unique_weight;
+                for lock_attempt in task.tx.1.iter() {
+                    lock_attempt.contended_unique_weights().insert_task(unique_weight, TaskInQueue::clone(&task));
+                }
+            }
+        }).unwrap();
 
         loop {
             trace!("schedule_once (from: {}, to: {}, runnnable: {}, contended: {}, (immediate+provisional)/max: ({}+{})/{}) active from contended: {}!", from.len(), to_execute_substage.len(), runnable_queue.task_count(), contended_count, executing_queue_count, address_book.provisioning_trackers.len(), max_executing_queue_count, address_book.uncontended_task_ids.len());

From 0a250a76e2ecf5a89edec08375598367685f91d8 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 25 Aug 2022 22:23:15 +0900
Subject: [PATCH 1349/3199] save

---
 scheduler/src/lib.rs | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 6cfb88d02ac1ef..b382ff1432e160 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -970,6 +970,7 @@ impl ScheduleStage {
 
     #[inline(never)]
     fn schedule_next_execution(
+        task_sender: usize,
         runnable_queue: &mut TaskQueue,
         address_book: &mut AddressBook,
         contended_count: &mut usize,
@@ -1094,7 +1095,7 @@ impl ScheduleStage {
 
                 let prefer_immediate = address_book.provisioning_trackers.len()/4 > executing_queue_count;
                 let maybe_ee =
-                    Self::schedule_next_execution(runnable_queue, address_book, &mut contended_count, prefer_immediate, &sequence_time, &mut queue_clock, &mut execute_clock);
+                    Self::schedule_next_execution(task_sender, runnable_queue, address_book, &mut contended_count, prefer_immediate, &sequence_time, &mut queue_clock, &mut execute_clock);
 
                 if let Some(ee) = maybe_ee {
                     trace!("send to execute");

From f8ac4d47eafc962cb7dfdc9062bcff0df03d1ad2 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 25 Aug 2022 22:23:36 +0900
Subject: [PATCH 1350/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index b382ff1432e160..1427fced172be6 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -970,7 +970,7 @@ impl ScheduleStage {
 
     #[inline(never)]
     fn schedule_next_execution(
-        task_sender: usize,
+        task_sender: crossbeam_channel::Sender<TaskInQueue>,
         runnable_queue: &mut TaskQueue,
         address_book: &mut AddressBook,
         contended_count: &mut usize,

From 1bbdba1d68522246fe077785d77377ec19999d2f Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 25 Aug 2022 22:23:44 +0900
Subject: [PATCH 1351/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 1427fced172be6..c7ac7484d6174b 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -970,7 +970,7 @@ impl ScheduleStage {
 
     #[inline(never)]
     fn schedule_next_execution(
-        task_sender: crossbeam_channel::Sender<TaskInQueue>,
+        task_sender: &crossbeam_channel::Sender<TaskInQueue>,
         runnable_queue: &mut TaskQueue,
         address_book: &mut AddressBook,
         contended_count: &mut usize,

From 98647e69e63e449c90966bb6dee125a9cb8d7986 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 25 Aug 2022 22:23:50 +0900
Subject: [PATCH 1352/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index c7ac7484d6174b..f0ad4836170abe 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1095,7 +1095,7 @@ impl ScheduleStage {
 
                 let prefer_immediate = address_book.provisioning_trackers.len()/4 > executing_queue_count;
                 let maybe_ee =
-                    Self::schedule_next_execution(task_sender, runnable_queue, address_book, &mut contended_count, prefer_immediate, &sequence_time, &mut queue_clock, &mut execute_clock);
+                    Self::schedule_next_execution(&task_sender, runnable_queue, address_book, &mut contended_count, prefer_immediate, &sequence_time, &mut queue_clock, &mut execute_clock);
 
                 if let Some(ee) = maybe_ee {
                     trace!("send to execute");

From 5b0ab7f5a4200e6b94fbc75472123b2b26e01b09 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 25 Aug 2022 22:24:45 +0900
Subject: [PATCH 1353/3199] save

---
 scheduler/src/lib.rs | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index f0ad4836170abe..0a158cb1266d09 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -720,6 +720,7 @@ impl ScheduleStage {
 
     #[inline(never)]
     fn pop_from_queue_then_lock(
+        task_sender: &crossbeam_channel::Sender<TaskInQueue>,
         runnable_queue: &mut TaskQueue,
         address_book: &mut AddressBook,
         contended_count: &mut usize,
@@ -776,9 +777,10 @@ impl ScheduleStage {
                     trace!("move to contended due to lock failure [{}/{}/{}]", unlockable_count, provisional_count, lock_count);
                     next_task.mark_as_contended();
                     *contended_count = contended_count.checked_add(1).unwrap();
-                    for lock_attempt in next_task.tx.1.iter() {
-                        lock_attempt.contended_unique_weights().insert_task(unique_weight, TaskInQueue::clone(&a2));
-                    }
+                    //for lock_attempt in next_task.tx.1.iter() {
+                    //    lock_attempt.contended_unique_weights().insert_task(unique_weight, TaskInQueue::clone(&a2));
+                    //}
+                    task_sender.send(TaskInQueue::clone(&a2)).unwrap();
                     // maybe run lightweight prune logic on contended_queue here.
                 } else {
                     trace!("relock failed [{}/{}/{}]; remains in contended: {:?} contention: {}", unlockable_count, provisional_count, lock_count, &unique_weight, next_task.contention_count);
@@ -980,7 +982,7 @@ impl ScheduleStage {
         execute_clock: &mut usize,
     ) -> Option<Box<ExecutionEnvironment>> {
         let maybe_ee =
-            Self::pop_from_queue_then_lock(runnable_queue, address_book, contended_count, prefer_immediate, sequence_time, queue_clock)
+            Self::pop_from_queue_then_lock(task_sender, runnable_queue, address_book, contended_count, prefer_immediate, sequence_time, queue_clock)
                 .map(|(uw, t,ll)| Self::prepare_scheduled_execution(address_book, uw, t, ll, queue_clock, execute_clock));
         maybe_ee
     }

From ef065823b0017fd9a923d2b854e4e545ff4b32ff Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 25 Aug 2022 22:27:17 +0900
Subject: [PATCH 1354/3199] save

---
 scheduler/src/lib.rs | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 0a158cb1266d09..926fdbe156ea42 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -153,7 +153,7 @@ impl TaskIds {
     #[inline(never)]
     pub fn remove_task(&self, u: &TaskId) {
         let removed_entry = self.task_ids.remove(u);
-        assert!(removed_entry.is_some());
+        //assert!(removed_entry.is_some());
     }
 
     #[inline(never)]
@@ -1034,9 +1034,11 @@ impl ScheduleStage {
         let (task_sender, task_receiver) = crossbeam_channel::unbounded::<TaskInQueue>();
         let h = std::thread::Builder::new().name("sol-indexer".to_string()).spawn(move || {
             while let task = task_receiver.recv().unwrap() {
-                let unique_weight = task.unique_weight;
-                for lock_attempt in task.tx.1.iter() {
-                    lock_attempt.contended_unique_weights().insert_task(unique_weight, TaskInQueue::clone(&task));
+                if task.currently_contended() {
+                    let unique_weight = task.unique_weight;
+                    for lock_attempt in task.tx.1.iter() {
+                        lock_attempt.contended_unique_weights().insert_task(unique_weight, TaskInQueue::clone(&task));
+                    }
                 }
             }
         }).unwrap();

From 8d7efbbc756770f6775ded3c8e41a153cdc303f3 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 25 Aug 2022 22:28:13 +0900
Subject: [PATCH 1355/3199] save

---
 scheduler/src/lib.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 926fdbe156ea42..a7b473b5653c94 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1034,12 +1034,12 @@ impl ScheduleStage {
         let (task_sender, task_receiver) = crossbeam_channel::unbounded::<TaskInQueue>();
         let h = std::thread::Builder::new().name("sol-indexer".to_string()).spawn(move || {
             while let task = task_receiver.recv().unwrap() {
-                if task.currently_contended() {
+                //if task.currently_contended() {
                     let unique_weight = task.unique_weight;
                     for lock_attempt in task.tx.1.iter() {
                         lock_attempt.contended_unique_weights().insert_task(unique_weight, TaskInQueue::clone(&task));
                     }
-                }
+                //}
             }
         }).unwrap();
 

From 574145b9764b52e6b0090a0e5ed41b70c6cb4b22 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 25 Aug 2022 22:31:23 +0900
Subject: [PATCH 1356/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index a7b473b5653c94..47f54a4630a81c 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -780,7 +780,7 @@ impl ScheduleStage {
                     //for lock_attempt in next_task.tx.1.iter() {
                     //    lock_attempt.contended_unique_weights().insert_task(unique_weight, TaskInQueue::clone(&a2));
                     //}
-                    task_sender.send(TaskInQueue::clone(&a2)).unwrap();
+                    task_sender.send(next_task.tx.1.clone()).unwrap();
                     // maybe run lightweight prune logic on contended_queue here.
                 } else {
                     trace!("relock failed [{}/{}/{}]; remains in contended: {:?} contention: {}", unlockable_count, provisional_count, lock_count, &unique_weight, next_task.contention_count);

From 237998a3bc679156051d901be915b27a28f83a4e Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 25 Aug 2022 22:31:49 +0900
Subject: [PATCH 1357/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 47f54a4630a81c..9e68941fc5d020 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -780,7 +780,7 @@ impl ScheduleStage {
                     //for lock_attempt in next_task.tx.1.iter() {
                     //    lock_attempt.contended_unique_weights().insert_task(unique_weight, TaskInQueue::clone(&a2));
                     //}
-                    task_sender.send(next_task.tx.1.clone()).unwrap();
+                    task_sender.send(next_task.tx.1.clone_for_test()).unwrap();
                     // maybe run lightweight prune logic on contended_queue here.
                 } else {
                     trace!("relock failed [{}/{}/{}]; remains in contended: {:?} contention: {}", unlockable_count, provisional_count, lock_count, &unique_weight, next_task.contention_count);

From 7b04c305c423282a79172e1fc76384be6ec55286 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 25 Aug 2022 22:33:45 +0900
Subject: [PATCH 1358/3199] save

---
 scheduler/src/lib.rs | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 9e68941fc5d020..b02aae08814f04 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -431,6 +431,7 @@ pub struct Task {
     pub queue_end_time: std::sync::atomic::AtomicUsize,
     pub execute_time: std::sync::atomic::AtomicUsize,
     pub commit_time: std::sync::atomic::AtomicUsize,
+    pub for_indexer: Vec<LockAttempt>,
 }
 
 // sequence_time -> seq clock
@@ -780,7 +781,7 @@ impl ScheduleStage {
                     //for lock_attempt in next_task.tx.1.iter() {
                     //    lock_attempt.contended_unique_weights().insert_task(unique_weight, TaskInQueue::clone(&a2));
                     //}
-                    task_sender.send(next_task.tx.1.clone_for_test()).unwrap();
+                    task_sender.send(TaskInQueue::clone(&a2)).unwrap();
                     // maybe run lightweight prune logic on contended_queue here.
                 } else {
                     trace!("relock failed [{}/{}/{}]; remains in contended: {:?} contention: {}", unlockable_count, provisional_count, lock_count, &unique_weight, next_task.contention_count);
@@ -1034,12 +1035,12 @@ impl ScheduleStage {
         let (task_sender, task_receiver) = crossbeam_channel::unbounded::<TaskInQueue>();
         let h = std::thread::Builder::new().name("sol-indexer".to_string()).spawn(move || {
             while let task = task_receiver.recv().unwrap() {
-                //if task.currently_contended() {
+                if task.currently_contended() {
                     let unique_weight = task.unique_weight;
                     for lock_attempt in task.tx.1.iter() {
                         lock_attempt.contended_unique_weights().insert_task(unique_weight, TaskInQueue::clone(&task));
                     }
-                //}
+                }
             }
         }).unwrap();
 

From 3a5c03048e26cde975d7571e47805e995c1e2e40 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 25 Aug 2022 22:34:25 +0900
Subject: [PATCH 1359/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index b02aae08814f04..6dede1a164a418 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -441,7 +441,7 @@ pub struct Task {
 
 impl Task {
     pub fn new_for_queue(unique_weight: UniqueWeight, tx: Box<(SanitizedTransaction, Vec<LockAttempt>)>) -> std::sync::Arc<Self> {
-        TaskInQueue::new(Self { unique_weight, tx, contention_count: 0, uncontended: Default::default(), sequence_time: std::sync::atomic::AtomicUsize::new(usize::max_value()), sequence_end_time: std::sync::atomic::AtomicUsize::new(usize::max_value()), queue_time: std::sync::atomic::AtomicUsize::new(usize::max_value()), queue_end_time: std::sync::atomic::AtomicUsize::new(usize::max_value()), execute_time: std::sync::atomic::AtomicUsize::new(usize::max_value()), commit_time: std::sync::atomic::AtomicUsize::new(usize::max_value()) })
+        TaskInQueue::new(Self { unique_weight, tx, contention_count: 0, uncontended: Default::default(), sequence_time: std::sync::atomic::AtomicUsize::new(usize::max_value()), sequence_end_time: std::sync::atomic::AtomicUsize::new(usize::max_value()), queue_time: std::sync::atomic::AtomicUsize::new(usize::max_value()), queue_end_time: std::sync::atomic::AtomicUsize::new(usize::max_value()), execute_time: std::sync::atomic::AtomicUsize::new(usize::max_value()), commit_time: std::sync::atomic::AtomicUsize::new(usize::max_value()), for_indexer: tx.1.iter().map(|a| a.clone_for_test()).collect() })
     }
 
     pub fn record_sequence_time(&self, clock: usize) {

From c8b22e5017ddc96a96e17a1e162ace8c461984ef Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 25 Aug 2022 22:35:21 +0900
Subject: [PATCH 1360/3199] save

---
 scheduler/src/lib.rs | 1 +
 1 file changed, 1 insertion(+)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 6dede1a164a418..4775b0e34bcc2e 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -516,6 +516,7 @@ impl Task {
             queue_end_time: std::sync::atomic::AtomicUsize::new(usize::max_value()),
             execute_time: std::sync::atomic::AtomicUsize::new(usize::max_value()),
             commit_time: std::sync::atomic::AtomicUsize::new(usize::max_value()),
+            for_indexer: tx.1.iter().map(|a| a.clone_for_test()).collect(),
         }
     }
 

From cabf7a04250f6ce6f6e90fe6704e777918883a65 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 25 Aug 2022 22:35:30 +0900
Subject: [PATCH 1361/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 4775b0e34bcc2e..385121ccdb9762 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -516,7 +516,7 @@ impl Task {
             queue_end_time: std::sync::atomic::AtomicUsize::new(usize::max_value()),
             execute_time: std::sync::atomic::AtomicUsize::new(usize::max_value()),
             commit_time: std::sync::atomic::AtomicUsize::new(usize::max_value()),
-            for_indexer: tx.1.iter().map(|a| a.clone_for_test()).collect(),
+            for_indexer: self.tx.1.iter().map(|a| a.clone_for_test()).collect(),
         }
     }
 

From 02dcacce0513c4f02d7ce069c9cb5497eb7f493e Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 25 Aug 2022 22:35:43 +0900
Subject: [PATCH 1362/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 385121ccdb9762..f48379d45858f8 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -507,6 +507,7 @@ impl Task {
     pub fn clone_for_test(&self) -> Self {
         Self {
             unique_weight: self.unique_weight,
+            for_indexer: self.tx.1.iter().map(|a| a.clone_for_test()).collect(),
             tx: Box::new((self.tx.0.clone(), self.tx.1.iter().map(|l| l.clone_for_test()).collect::<Vec<_>>())),
             contention_count: Default::default(),
             uncontended: Default::default(),
@@ -516,7 +517,6 @@ impl Task {
             queue_end_time: std::sync::atomic::AtomicUsize::new(usize::max_value()),
             execute_time: std::sync::atomic::AtomicUsize::new(usize::max_value()),
             commit_time: std::sync::atomic::AtomicUsize::new(usize::max_value()),
-            for_indexer: self.tx.1.iter().map(|a| a.clone_for_test()).collect(),
         }
     }
 

From 810bd0655c898caa91bdf0627390770ca57be746 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 25 Aug 2022 22:36:08 +0900
Subject: [PATCH 1363/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index f48379d45858f8..2fbda72e656804 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -441,7 +441,7 @@ pub struct Task {
 
 impl Task {
     pub fn new_for_queue(unique_weight: UniqueWeight, tx: Box<(SanitizedTransaction, Vec<LockAttempt>)>) -> std::sync::Arc<Self> {
-        TaskInQueue::new(Self { unique_weight, tx, contention_count: 0, uncontended: Default::default(), sequence_time: std::sync::atomic::AtomicUsize::new(usize::max_value()), sequence_end_time: std::sync::atomic::AtomicUsize::new(usize::max_value()), queue_time: std::sync::atomic::AtomicUsize::new(usize::max_value()), queue_end_time: std::sync::atomic::AtomicUsize::new(usize::max_value()), execute_time: std::sync::atomic::AtomicUsize::new(usize::max_value()), commit_time: std::sync::atomic::AtomicUsize::new(usize::max_value()), for_indexer: tx.1.iter().map(|a| a.clone_for_test()).collect() })
+        TaskInQueue::new(Self { for_indexer: tx.1.iter().map(|a| a.clone_for_test()).collect(), unique_weight, tx, contention_count: 0, uncontended: Default::default(), sequence_time: std::sync::atomic::AtomicUsize::new(usize::max_value()), sequence_end_time: std::sync::atomic::AtomicUsize::new(usize::max_value()), queue_time: std::sync::atomic::AtomicUsize::new(usize::max_value()), queue_end_time: std::sync::atomic::AtomicUsize::new(usize::max_value()), execute_time: std::sync::atomic::AtomicUsize::new(usize::max_value()), commit_time: std::sync::atomic::AtomicUsize::new(usize::max_value()),  })
     }
 
     pub fn record_sequence_time(&self, clock: usize) {

From 07747382741757d3defe77ee62899b465d1d5dac Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 25 Aug 2022 22:36:42 +0900
Subject: [PATCH 1364/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 2fbda72e656804..eeb73281882adf 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -782,7 +782,7 @@ impl ScheduleStage {
                     //for lock_attempt in next_task.tx.1.iter() {
                     //    lock_attempt.contended_unique_weights().insert_task(unique_weight, TaskInQueue::clone(&a2));
                     //}
-                    task_sender.send(TaskInQueue::clone(&a2)).unwrap();
+                    task_sender.send(std::mem::take(&next_task.for_indexer)).unwrap();
                     // maybe run lightweight prune logic on contended_queue here.
                 } else {
                     trace!("relock failed [{}/{}/{}]; remains in contended: {:?} contention: {}", unlockable_count, provisional_count, lock_count, &unique_weight, next_task.contention_count);

From f39262b5a527337f70a257034518e62f528b8c8f Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 25 Aug 2022 22:37:01 +0900
Subject: [PATCH 1365/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index eeb73281882adf..334f195cca64ba 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -782,7 +782,7 @@ impl ScheduleStage {
                     //for lock_attempt in next_task.tx.1.iter() {
                     //    lock_attempt.contended_unique_weights().insert_task(unique_weight, TaskInQueue::clone(&a2));
                     //}
-                    task_sender.send(std::mem::take(&next_task.for_indexer)).unwrap();
+                    task_sender.send(*std::mem::take(&next_task.for_indexer)).unwrap();
                     // maybe run lightweight prune logic on contended_queue here.
                 } else {
                     trace!("relock failed [{}/{}/{}]; remains in contended: {:?} contention: {}", unlockable_count, provisional_count, lock_count, &unique_weight, next_task.contention_count);

From 719281f7d42b9b2810db0fed9402decfabb18b29 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 25 Aug 2022 22:37:09 +0900
Subject: [PATCH 1366/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 334f195cca64ba..d75bf92df4c805 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -782,7 +782,7 @@ impl ScheduleStage {
                     //for lock_attempt in next_task.tx.1.iter() {
                     //    lock_attempt.contended_unique_weights().insert_task(unique_weight, TaskInQueue::clone(&a2));
                     //}
-                    task_sender.send(*std::mem::take(&next_task.for_indexer)).unwrap();
+                    task_sender.send(*std::mem::take(&mut next_task.for_indexer)).unwrap();
                     // maybe run lightweight prune logic on contended_queue here.
                 } else {
                     trace!("relock failed [{}/{}/{}]; remains in contended: {:?} contention: {}", unlockable_count, provisional_count, lock_count, &unique_weight, next_task.contention_count);

From a58c4027295d23b7c0800ccf5cdda87e39751f5e Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 25 Aug 2022 22:37:18 +0900
Subject: [PATCH 1367/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index d75bf92df4c805..a541330337e6fe 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -782,7 +782,7 @@ impl ScheduleStage {
                     //for lock_attempt in next_task.tx.1.iter() {
                     //    lock_attempt.contended_unique_weights().insert_task(unique_weight, TaskInQueue::clone(&a2));
                     //}
-                    task_sender.send(*std::mem::take(&mut next_task.for_indexer)).unwrap();
+                    task_sender.send(std::mem::take(&mut next_task.for_indexer)).unwrap();
                     // maybe run lightweight prune logic on contended_queue here.
                 } else {
                     trace!("relock failed [{}/{}/{}]; remains in contended: {:?} contention: {}", unlockable_count, provisional_count, lock_count, &unique_weight, next_task.contention_count);

From f2e5be09b3e137404fac240a06ec3146110949d4 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 25 Aug 2022 22:37:38 +0900
Subject: [PATCH 1368/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index a541330337e6fe..cada2048ec38b5 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -722,7 +722,7 @@ impl ScheduleStage {
 
     #[inline(never)]
     fn pop_from_queue_then_lock(
-        task_sender: &crossbeam_channel::Sender<TaskInQueue>,
+        task_sender: &crossbeam_channel::Sender<Vec<LockAttempt>>,
         runnable_queue: &mut TaskQueue,
         address_book: &mut AddressBook,
         contended_count: &mut usize,

From e6007cd24828bdb25933b7549493be1b742ed42d Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 25 Aug 2022 22:37:56 +0900
Subject: [PATCH 1369/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index cada2048ec38b5..ae1988c58e833a 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -974,7 +974,7 @@ impl ScheduleStage {
 
     #[inline(never)]
     fn schedule_next_execution(
-        task_sender: &crossbeam_channel::Sender<TaskInQueue>,
+        task_sender: &crossbeam_channel::Sender<Vec<LockAttempt>>,
         runnable_queue: &mut TaskQueue,
         address_book: &mut AddressBook,
         contended_count: &mut usize,

From 99a14c2b4fb19642a537e8ec65104dac287a4ec1 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 25 Aug 2022 22:38:12 +0900
Subject: [PATCH 1370/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index ae1988c58e833a..03e55667dd6081 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1033,7 +1033,7 @@ impl ScheduleStage {
 
             (&ee_sender, Some(h))
         };
-        let (task_sender, task_receiver) = crossbeam_channel::unbounded::<TaskInQueue>();
+        let (task_sender, task_receiver) = crossbeam_channel::unbounded::<Vec<LockAttempt>>();
         let h = std::thread::Builder::new().name("sol-indexer".to_string()).spawn(move || {
             while let task = task_receiver.recv().unwrap() {
                 if task.currently_contended() {

From c66ae6218d0fae757f014b77a7c4504582318c96 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 25 Aug 2022 22:41:55 +0900
Subject: [PATCH 1371/3199] save

---
 scheduler/src/lib.rs | 8 +++-----
 1 file changed, 3 insertions(+), 5 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 03e55667dd6081..b564bed7a828a5 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -782,7 +782,7 @@ impl ScheduleStage {
                     //for lock_attempt in next_task.tx.1.iter() {
                     //    lock_attempt.contended_unique_weights().insert_task(unique_weight, TaskInQueue::clone(&a2));
                     //}
-                    task_sender.send(std::mem::take(&mut next_task.for_indexer)).unwrap();
+                    task_sender.send((*unique_weight, TaskInQueue::clone(&a2), std::mem::take(&mut next_task.for_indexer))).unwrap();
                     // maybe run lightweight prune logic on contended_queue here.
                 } else {
                     trace!("relock failed [{}/{}/{}]; remains in contended: {:?} contention: {}", unlockable_count, provisional_count, lock_count, &unique_weight, next_task.contention_count);
@@ -1035,11 +1035,9 @@ impl ScheduleStage {
         };
         let (task_sender, task_receiver) = crossbeam_channel::unbounded::<Vec<LockAttempt>>();
         let h = std::thread::Builder::new().name("sol-indexer".to_string()).spawn(move || {
-            while let task = task_receiver.recv().unwrap() {
-                if task.currently_contended() {
-                    let unique_weight = task.unique_weight;
+            while let (uw, ll) = task_receiver.recv().unwrap() {
                     for lock_attempt in task.tx.1.iter() {
-                        lock_attempt.contended_unique_weights().insert_task(unique_weight, TaskInQueue::clone(&task));
+                        lock_attempt.contended_unique_weights().insert_task(uw, TaskInQueue::clone(&task));
                     }
                 }
             }

From c292f39a9e6333f7ad2a4a72ae6f055fa8172e01 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 25 Aug 2022 22:42:32 +0900
Subject: [PATCH 1372/3199] save

---
 scheduler/src/lib.rs | 1 -
 1 file changed, 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index b564bed7a828a5..95206941705c74 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1039,7 +1039,6 @@ impl ScheduleStage {
                     for lock_attempt in task.tx.1.iter() {
                         lock_attempt.contended_unique_weights().insert_task(uw, TaskInQueue::clone(&task));
                     }
-                }
             }
         }).unwrap();
 

From 0e0b4f295914b9e7379fa64cc76a84ebd7c543c9 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 25 Aug 2022 22:42:55 +0900
Subject: [PATCH 1373/3199] save

---
 scheduler/src/lib.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 95206941705c74..b62bdb71c2f1fc 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1035,11 +1035,11 @@ impl ScheduleStage {
         };
         let (task_sender, task_receiver) = crossbeam_channel::unbounded::<Vec<LockAttempt>>();
         let h = std::thread::Builder::new().name("sol-indexer".to_string()).spawn(move || {
-            while let (uw, ll) = task_receiver.recv().unwrap() {
+            /*while let (uw, ll) = task_receiver.recv().unwrap() {
                     for lock_attempt in task.tx.1.iter() {
                         lock_attempt.contended_unique_weights().insert_task(uw, TaskInQueue::clone(&task));
                     }
-            }
+            }*/
         }).unwrap();
 
         loop {

From e737b5ca968266ae5eeb3b075145055b0b5709da Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 25 Aug 2022 22:43:13 +0900
Subject: [PATCH 1374/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index b62bdb71c2f1fc..6f6c98c5fa129d 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -782,7 +782,7 @@ impl ScheduleStage {
                     //for lock_attempt in next_task.tx.1.iter() {
                     //    lock_attempt.contended_unique_weights().insert_task(unique_weight, TaskInQueue::clone(&a2));
                     //}
-                    task_sender.send((*unique_weight, TaskInQueue::clone(&a2), std::mem::take(&mut next_task.for_indexer))).unwrap();
+                    task_sender.send((unique_weight, TaskInQueue::clone(&a2), std::mem::take(&mut next_task.for_indexer))).unwrap();
                     // maybe run lightweight prune logic on contended_queue here.
                 } else {
                     trace!("relock failed [{}/{}/{}]; remains in contended: {:?} contention: {}", unlockable_count, provisional_count, lock_count, &unique_weight, next_task.contention_count);

From 2ec13b14857a6e598440880758f2839b18025725 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 25 Aug 2022 22:43:47 +0900
Subject: [PATCH 1375/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 6f6c98c5fa129d..b4f7b3b467c7e7 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -974,7 +974,7 @@ impl ScheduleStage {
 
     #[inline(never)]
     fn schedule_next_execution(
-        task_sender: &crossbeam_channel::Sender<Vec<LockAttempt>>,
+        task_sender: &crossbeam_channel::Sender<(u64, Arc<Task>, Vec<LockAttempt>)>,
         runnable_queue: &mut TaskQueue,
         address_book: &mut AddressBook,
         contended_count: &mut usize,

From 38806c2d64599f5cd64c155386a7cfc1bbfa86d3 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 25 Aug 2022 22:43:59 +0900
Subject: [PATCH 1376/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index b4f7b3b467c7e7..fb624d461fd9e4 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -974,7 +974,7 @@ impl ScheduleStage {
 
     #[inline(never)]
     fn schedule_next_execution(
-        task_sender: &crossbeam_channel::Sender<(u64, Arc<Task>, Vec<LockAttempt>)>,
+        task_sender: &crossbeam_channel::Sender<(u64, TaskInQueue, Vec<LockAttempt>)>,
         runnable_queue: &mut TaskQueue,
         address_book: &mut AddressBook,
         contended_count: &mut usize,

From 3bdcbf0cf51ec77a5ca61dab5ef6a368c243d146 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 25 Aug 2022 22:44:27 +0900
Subject: [PATCH 1377/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index fb624d461fd9e4..2ee36651640b81 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -722,7 +722,7 @@ impl ScheduleStage {
 
     #[inline(never)]
     fn pop_from_queue_then_lock(
-        task_sender: &crossbeam_channel::Sender<Vec<LockAttempt>>,
+        task_sender: &crossbeam_channel::Sender<(u64, TaskInQueue, Vec<LockAttempt>)>,
         runnable_queue: &mut TaskQueue,
         address_book: &mut AddressBook,
         contended_count: &mut usize,

From 0f63d42995d1524bbf2147120fd0036e1ce9146a Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 25 Aug 2022 22:45:05 +0900
Subject: [PATCH 1378/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 2ee36651640b81..9076429925fdf6 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1033,7 +1033,7 @@ impl ScheduleStage {
 
             (&ee_sender, Some(h))
         };
-        let (task_sender, task_receiver) = crossbeam_channel::unbounded::<Vec<LockAttempt>>();
+        let (task_sender, task_receiver) = crossbeam_channel::unbounded::<(u64, TaskInQueue, Vec<LockAttempt>)>();
         let h = std::thread::Builder::new().name("sol-indexer".to_string()).spawn(move || {
             /*while let (uw, ll) = task_receiver.recv().unwrap() {
                     for lock_attempt in task.tx.1.iter() {

From eeef13826668bc794e5454e0b1cfeecebb28917d Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 25 Aug 2022 22:45:43 +0900
Subject: [PATCH 1379/3199] save

---
 scheduler/src/lib.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 9076429925fdf6..31a6b294b80562 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1035,11 +1035,11 @@ impl ScheduleStage {
         };
         let (task_sender, task_receiver) = crossbeam_channel::unbounded::<(u64, TaskInQueue, Vec<LockAttempt>)>();
         let h = std::thread::Builder::new().name("sol-indexer".to_string()).spawn(move || {
-            /*while let (uw, ll) = task_receiver.recv().unwrap() {
+            while let (uw, ll) = task_receiver.recv().unwrap() {
                     for lock_attempt in task.tx.1.iter() {
                         lock_attempt.contended_unique_weights().insert_task(uw, TaskInQueue::clone(&task));
                     }
-            }*/
+            }
         }).unwrap();
 
         loop {

From f90539a264f85d4e7cbab14dfe319f574d0fd772 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 25 Aug 2022 22:46:09 +0900
Subject: [PATCH 1380/3199] save

---
 scheduler/src/lib.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 31a6b294b80562..2e08732043fe2b 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1035,8 +1035,8 @@ impl ScheduleStage {
         };
         let (task_sender, task_receiver) = crossbeam_channel::unbounded::<(u64, TaskInQueue, Vec<LockAttempt>)>();
         let h = std::thread::Builder::new().name("sol-indexer".to_string()).spawn(move || {
-            while let (uw, ll) = task_receiver.recv().unwrap() {
-                    for lock_attempt in task.tx.1.iter() {
+            while let (uw, task, ll) = task_receiver.recv().unwrap() {
+                    for lock_attempt in ll {
                         lock_attempt.contended_unique_weights().insert_task(uw, TaskInQueue::clone(&task));
                     }
             }

From f3029d64579d8e30b856a9d18cf1077a734ee259 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 25 Aug 2022 22:46:58 +0900
Subject: [PATCH 1381/3199] save

---
 scheduler/src/lib.rs | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 2e08732043fe2b..134e12a54fe415 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1036,9 +1036,9 @@ impl ScheduleStage {
         let (task_sender, task_receiver) = crossbeam_channel::unbounded::<(u64, TaskInQueue, Vec<LockAttempt>)>();
         let h = std::thread::Builder::new().name("sol-indexer".to_string()).spawn(move || {
             while let (uw, task, ll) = task_receiver.recv().unwrap() {
-                    for lock_attempt in ll {
-                        lock_attempt.contended_unique_weights().insert_task(uw, TaskInQueue::clone(&task));
-                    }
+                for lock_attempt in ll {
+                    lock_attempt.contended_unique_weights().insert_task(uw, TaskInQueue::clone(&task));
+                }
             }
         }).unwrap();
 

From aa47f6565257ddda3c8bbb96ded418f63eb0ca00 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 25 Aug 2022 22:50:31 +0900
Subject: [PATCH 1382/3199] save

---
 transaction-scheduler-bench/src/main.rs | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/transaction-scheduler-bench/src/main.rs b/transaction-scheduler-bench/src/main.rs
index 888aac7a999740..80d60e53e9adf0 100644
--- a/transaction-scheduler-bench/src/main.rs
+++ b/transaction-scheduler-bench/src/main.rs
@@ -369,6 +369,9 @@ fn handle_transaction_batch(
                     removed = task_cursor.remove();
                     assert!(removed);
                 }
+                if task.already_finished() {
+                    panic!()
+                }
                 if let Some(new_cursor) = task_cursor.prev() {
                     assert!(new_cursor.key() < task_cursor.key());
                     task_cursor = new_cursor;

From ca6629c7cd6a60a10f3ed2f8ca76cfbb517983eb Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 25 Aug 2022 22:51:08 +0900
Subject: [PATCH 1383/3199] save

---
 transaction-scheduler-bench/src/main.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/transaction-scheduler-bench/src/main.rs b/transaction-scheduler-bench/src/main.rs
index 80d60e53e9adf0..8e806165b5efa1 100644
--- a/transaction-scheduler-bench/src/main.rs
+++ b/transaction-scheduler-bench/src/main.rs
@@ -370,7 +370,7 @@ fn handle_transaction_batch(
                     assert!(removed);
                 }
                 if task.already_finished() {
-                    panic!()
+                    task_cursor.remove();
                 }
                 if let Some(new_cursor) = task_cursor.prev() {
                     assert!(new_cursor.key() < task_cursor.key());

From 8b5bbee2a90849dd9f572a826947204b5f7bde61 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 25 Aug 2022 22:55:54 +0900
Subject: [PATCH 1384/3199] save

---
 scheduler/src/lib.rs | 14 ++++++++------
 1 file changed, 8 insertions(+), 6 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 134e12a54fe415..01e2e4e3c71dd7 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1034,13 +1034,15 @@ impl ScheduleStage {
             (&ee_sender, Some(h))
         };
         let (task_sender, task_receiver) = crossbeam_channel::unbounded::<(u64, TaskInQueue, Vec<LockAttempt>)>();
-        let h = std::thread::Builder::new().name("sol-indexer".to_string()).spawn(move || {
-            while let (uw, task, ll) = task_receiver.recv().unwrap() {
-                for lock_attempt in ll {
-                    lock_attempt.contended_unique_weights().insert_task(uw, TaskInQueue::clone(&task));
+        for thx in 0..4 {
+            let h = std::thread::Builder::new().name(format!("sol-indexer{:02}", thx)).spawn(move || {
+                while let (uw, task, ll) = task_receiver.recv().unwrap() {
+                    for lock_attempt in ll {
+                        lock_attempt.contended_unique_weights().insert_task(uw, TaskInQueue::clone(&task));
+                    }
                 }
-            }
-        }).unwrap();
+            }).unwrap();
+        }
 
         loop {
             trace!("schedule_once (from: {}, to: {}, runnnable: {}, contended: {}, (immediate+provisional)/max: ({}+{})/{}) active from contended: {}!", from.len(), to_execute_substage.len(), runnable_queue.task_count(), contended_count, executing_queue_count, address_book.provisioning_trackers.len(), max_executing_queue_count, address_book.uncontended_task_ids.len());

From 89573f7bd9cd2615967393f262a37b0307dbfb20 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 25 Aug 2022 22:56:10 +0900
Subject: [PATCH 1385/3199] save

---
 scheduler/src/lib.rs | 1 +
 1 file changed, 1 insertion(+)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 01e2e4e3c71dd7..9b007723555a50 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1036,6 +1036,7 @@ impl ScheduleStage {
         let (task_sender, task_receiver) = crossbeam_channel::unbounded::<(u64, TaskInQueue, Vec<LockAttempt>)>();
         for thx in 0..4 {
             let h = std::thread::Builder::new().name(format!("sol-indexer{:02}", thx)).spawn(move || {
+                let task_receiver.clone();
                 while let (uw, task, ll) = task_receiver.recv().unwrap() {
                     for lock_attempt in ll {
                         lock_attempt.contended_unique_weights().insert_task(uw, TaskInQueue::clone(&task));

From 4ec11f5d557d6bd6cfbe0aefd011c24a0f6dce9c Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 25 Aug 2022 22:56:20 +0900
Subject: [PATCH 1386/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 9b007723555a50..87f5952310efc1 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1036,7 +1036,7 @@ impl ScheduleStage {
         let (task_sender, task_receiver) = crossbeam_channel::unbounded::<(u64, TaskInQueue, Vec<LockAttempt>)>();
         for thx in 0..4 {
             let h = std::thread::Builder::new().name(format!("sol-indexer{:02}", thx)).spawn(move || {
-                let task_receiver.clone();
+                let task_receiver = task_receiver.clone();
                 while let (uw, task, ll) = task_receiver.recv().unwrap() {
                     for lock_attempt in ll {
                         lock_attempt.contended_unique_weights().insert_task(uw, TaskInQueue::clone(&task));

From 5be3073db9a7865dcd4bd7b9cb806f4989780623 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 25 Aug 2022 22:56:42 +0900
Subject: [PATCH 1387/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 87f5952310efc1..68af4a271ad483 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1035,8 +1035,8 @@ impl ScheduleStage {
         };
         let (task_sender, task_receiver) = crossbeam_channel::unbounded::<(u64, TaskInQueue, Vec<LockAttempt>)>();
         for thx in 0..4 {
+            let task_receiver = task_receiver.clone();
             let h = std::thread::Builder::new().name(format!("sol-indexer{:02}", thx)).spawn(move || {
-                let task_receiver = task_receiver.clone();
                 while let (uw, task, ll) = task_receiver.recv().unwrap() {
                     for lock_attempt in ll {
                         lock_attempt.contended_unique_weights().insert_task(uw, TaskInQueue::clone(&task));

From d2ac0839acdaf3809a6b2d4ba620951f92754e95 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 25 Aug 2022 22:58:28 +0900
Subject: [PATCH 1388/3199] save

---
 scheduler/src/lib.rs                    | 6 +++++-
 transaction-scheduler-bench/src/main.rs | 6 +++---
 2 files changed, 8 insertions(+), 4 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 68af4a271ad483..5dd0a9f05e9866 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1034,7 +1034,11 @@ impl ScheduleStage {
             (&ee_sender, Some(h))
         };
         let (task_sender, task_receiver) = crossbeam_channel::unbounded::<(u64, TaskInQueue, Vec<LockAttempt>)>();
-        for thx in 0..4 {
+        let indexer_count = std::env::var("INDEXER_COUNT")
+            .unwrap_or(format!("{}", 4))
+            .parse::<usize>()
+            .unwrap();
+        for thx in 0..indexer_count {
             let task_receiver = task_receiver.clone();
             let h = std::thread::Builder::new().name(format!("sol-indexer{:02}", thx)).spawn(move || {
                 while let (uw, task, ll) = task_receiver.recv().unwrap() {
diff --git a/transaction-scheduler-bench/src/main.rs b/transaction-scheduler-bench/src/main.rs
index 8e806165b5efa1..4bf6d8df9b2179 100644
--- a/transaction-scheduler-bench/src/main.rs
+++ b/transaction-scheduler-bench/src/main.rs
@@ -362,7 +362,7 @@ fn handle_transaction_batch(
             let mut found = true;
             let mut removed = false;
             let mut task = task_cursor.value();
-            task.trace_timestamps("in_exec(initial list)");
+            //task.trace_timestamps("in_exec(initial list)");
             while !task.currently_contended() {
                 if task_cursor.key() == &uq {
                     assert!(should_remove);
@@ -376,7 +376,7 @@ fn handle_transaction_batch(
                     assert!(new_cursor.key() < task_cursor.key());
                     task_cursor = new_cursor;
                     task = task_cursor.value();
-                    task.trace_timestamps("in_exec(subsequent list)");
+                    //task.trace_timestamps("in_exec(subsequent list)");
                 } else {
                     found = false;
                     break;
@@ -387,7 +387,7 @@ fn handle_transaction_batch(
             }
             found.then(|| solana_scheduler::TaskInQueue::clone(task))
         }).flatten().map(|task| {
-            task.trace_timestamps(&format!("in_exec(heaviest:{})", transaction_batch.task.queue_time_label()));
+            //task.trace_timestamps(&format!("in_exec(heaviest:{})", transaction_batch.task.queue_time_label()));
             lock_attempt.heaviest_uncontended = Some(task);
             ()
         });

From 149bcd3620ce60c0a86374595e12d0f833caccc9 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 26 Aug 2022 09:01:31 +0900
Subject: [PATCH 1389/3199] save

---
 transaction-scheduler-bench/src/main.rs | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/transaction-scheduler-bench/src/main.rs b/transaction-scheduler-bench/src/main.rs
index 4bf6d8df9b2179..950f9847deba94 100644
--- a/transaction-scheduler-bench/src/main.rs
+++ b/transaction-scheduler-bench/src/main.rs
@@ -127,11 +127,16 @@ fn spawn_unified_scheduler(
         max_batch_size: usize,
         exit: Arc<AtomicBool>,
 ) -> JoinHandle<()> {
+    let lane_channel_factor = std::env::var("LANE_CHANNEL_FACTOR")
+        .unwrap_or(format!("{}", 10)
+        .parse::<usize>()
+        .unwrap();
+
     std::thread::Builder::new().name("sol-scheduler".to_string()).spawn(move || {
         let mut runnable_queue = solana_scheduler::TaskQueue::default();
 
         solana_scheduler::ScheduleStage::run(
-            num_execution_threads * 10,
+            num_execution_threads * lane_channel_factor,
             &mut runnable_queue,
             &mut address_book,
             &packet_batch_receiver.clone(),

From d7f7681f31d81ea4e72934cfe5ce7c8af96dd823 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 26 Aug 2022 09:02:15 +0900
Subject: [PATCH 1390/3199] save

---
 transaction-scheduler-bench/src/main.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/transaction-scheduler-bench/src/main.rs b/transaction-scheduler-bench/src/main.rs
index 950f9847deba94..60dc4cda6ca70a 100644
--- a/transaction-scheduler-bench/src/main.rs
+++ b/transaction-scheduler-bench/src/main.rs
@@ -128,7 +128,7 @@ fn spawn_unified_scheduler(
         exit: Arc<AtomicBool>,
 ) -> JoinHandle<()> {
     let lane_channel_factor = std::env::var("LANE_CHANNEL_FACTOR")
-        .unwrap_or(format!("{}", 10)
+        .unwrap_or(format!("{}", 10)))
         .parse::<usize>()
         .unwrap();
 

From f8c0fffce7702504b077d898f8adc49214daa50a Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 26 Aug 2022 09:02:19 +0900
Subject: [PATCH 1391/3199] save

---
 transaction-scheduler-bench/src/main.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/transaction-scheduler-bench/src/main.rs b/transaction-scheduler-bench/src/main.rs
index 60dc4cda6ca70a..5e092bab3a0e40 100644
--- a/transaction-scheduler-bench/src/main.rs
+++ b/transaction-scheduler-bench/src/main.rs
@@ -128,7 +128,7 @@ fn spawn_unified_scheduler(
         exit: Arc<AtomicBool>,
 ) -> JoinHandle<()> {
     let lane_channel_factor = std::env::var("LANE_CHANNEL_FACTOR")
-        .unwrap_or(format!("{}", 10)))
+        .unwrap_or(format!("{}", 10))
         .parse::<usize>()
         .unwrap();
 

From 2db896583d5917f9dfb2991dcf525e245ba4a210 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 26 Aug 2022 12:48:45 +0900
Subject: [PATCH 1392/3199] save

---
 scheduler/src/lib.rs | 36 +++++++-----------------------------
 1 file changed, 7 insertions(+), 29 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 5dd0a9f05e9866..e3eb6454e7155c 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1058,19 +1058,7 @@ impl ScheduleStage {
                     match i {
                         Multiplexed::FromPrevious(weighted_tx) => {
                             trace!("recv from previous");
-
                             Self::register_runnable_task(weighted_tx, runnable_queue, &mut current_unique_key, &mut sequence_time);
-
-                            while false && from_exec.len() > 0 {
-                                let mut processed_execution_environment = from_exec.recv().unwrap();
-                                trace!("recv from execute: {:?}", processed_execution_environment.unique_weight);
-                                executing_queue_count -= 1;
-
-                                Self::commit_completed_execution(&mut processed_execution_environment, address_book, &mut execute_clock);
-                                // async-ly propagate the result to rpc subsystems
-                                to_next_stage.send(processed_execution_environment).unwrap();
-                            }
-
                         }
                         Multiplexed::FromPreviousBatched(vvv) => {
                             unreachable!();
@@ -1079,26 +1067,17 @@ impl ScheduleStage {
                }
                recv(from_exec) -> maybe_from_exec => {
                    let mut processed_execution_environment = maybe_from_exec.unwrap();
-                    loop {
-                        trace!("recv from execute: {:?}", processed_execution_environment.unique_weight);
-                        executing_queue_count -= 1;
-
-                        Self::commit_completed_execution(&mut processed_execution_environment, address_book, &mut execute_clock);
-                        // async-ly propagate the result to rpc subsystems
-                        to_next_stage.send(processed_execution_environment).unwrap();
-                        if false && from_exec.len() > 0 {
-                            processed_execution_environment = from_exec.recv().unwrap();
-                        } else {
-                            break;
-                        }
-                    }
+                    trace!("recv from execute: {:?}", processed_execution_environment.unique_weight);
+                    executing_queue_count -= 1;
+
+                    Self::commit_completed_execution(&mut processed_execution_environment, address_book, &mut execute_clock);
+                    // async-ly propagate the result to rpc subsystems
+                    to_next_stage.send(processed_execution_environment).unwrap();
                }
             }
 
             loop {
-                /* if !address_book.uncontended_task_ids.is_empty() {
-                    trace!("prefer emptying n_u_a");
-                } else */ if (executing_queue_count + address_book.provisioning_trackers.len()) >= max_executing_queue_count {
+                if (executing_queue_count + address_book.provisioning_trackers.len()) >= max_executing_queue_count {
                     trace!("skip scheduling; outgoing queue full");
                     break;
                 }
@@ -1112,7 +1091,6 @@ impl ScheduleStage {
                     executing_queue_count += 1;
 
                     to_execute_substage.send(ee).unwrap();
-                    break;
                 } else {
                     break;
                 }

From 485cc58dda6b6bf06007a63cc168ac8ae711b28d Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 26 Aug 2022 12:50:58 +0900
Subject: [PATCH 1393/3199] save

---
 scheduler/src/lib.rs | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index e3eb6454e7155c..4d95093e94d6fc 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -617,7 +617,6 @@ type PreprocessedTransaction = (SanitizedTransaction, Vec<LockAttempt>);
 // switched from crossbeam_channel::select! due to observed poor performance
 pub enum Multiplexed {
     FromPrevious((Weight, TaskInQueue)),
-    FromPreviousBatched(Vec<Vec<Box<PreprocessedTransaction>>>),
 }
 
 pub fn get_transaction_priority_details(tx: &SanitizedTransaction) -> u64 {
@@ -1060,9 +1059,6 @@ impl ScheduleStage {
                             trace!("recv from previous");
                             Self::register_runnable_task(weighted_tx, runnable_queue, &mut current_unique_key, &mut sequence_time);
                         }
-                        Multiplexed::FromPreviousBatched(vvv) => {
-                            unreachable!();
-                        }
                     }
                }
                recv(from_exec) -> maybe_from_exec => {

From 6297c53db515f52124ed13361c251a35d326f165 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 26 Aug 2022 12:53:18 +0900
Subject: [PATCH 1394/3199] save

---
 scheduler/src/lib.rs | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 4d95093e94d6fc..9cb4ccfd2c1c70 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1053,6 +1053,7 @@ impl ScheduleStage {
 
             crossbeam_channel::select! {
                recv(from) -> maybe_from => {
+                   error!("select1: {} {}", from.len(), from_exec.len());
                    let i = maybe_from.unwrap();
                     match i {
                         Multiplexed::FromPrevious(weighted_tx) => {
@@ -1062,6 +1063,7 @@ impl ScheduleStage {
                     }
                }
                recv(from_exec) -> maybe_from_exec => {
+                   error!("select2: {} {}", from.len(), from_exec.len());
                    let mut processed_execution_environment = maybe_from_exec.unwrap();
                     trace!("recv from execute: {:?}", processed_execution_environment.unique_weight);
                     executing_queue_count -= 1;

From 953607fd735f5645d847461776286e4bf823169e Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 26 Aug 2022 12:54:14 +0900
Subject: [PATCH 1395/3199] save

---
 scheduler/src/lib.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 9cb4ccfd2c1c70..d5920943f89d89 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1053,7 +1053,7 @@ impl ScheduleStage {
 
             crossbeam_channel::select! {
                recv(from) -> maybe_from => {
-                   error!("select1: {} {}", from.len(), from_exec.len());
+                   trace!("select1: {} {}", from.len(), from_exec.len());
                    let i = maybe_from.unwrap();
                     match i {
                         Multiplexed::FromPrevious(weighted_tx) => {
@@ -1063,7 +1063,7 @@ impl ScheduleStage {
                     }
                }
                recv(from_exec) -> maybe_from_exec => {
-                   error!("select2: {} {}", from.len(), from_exec.len());
+                   trace!("select2: {} {}", from.len(), from_exec.len());
                    let mut processed_execution_environment = maybe_from_exec.unwrap();
                     trace!("recv from execute: {:?}", processed_execution_environment.unique_weight);
                     executing_queue_count -= 1;

From 4a892f9a0c00f35b8cd19407fe86ac2fb08af767 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 26 Aug 2022 12:57:19 +0900
Subject: [PATCH 1396/3199] save

---
 scheduler/src/lib.rs | 17 +++--------------
 1 file changed, 3 insertions(+), 14 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index d5920943f89d89..ce495ccb07bc6f 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -612,12 +612,6 @@ fn attempt_lock_for_execution<'a>(
 }
 
 type PreprocessedTransaction = (SanitizedTransaction, Vec<LockAttempt>);
-// multiplexed to reduce the futex syscal per tx down to minimum and to make the schduler to
-// adaptive relative load between sigverify stage and execution substage
-// switched from crossbeam_channel::select! due to observed poor performance
-pub enum Multiplexed {
-    FromPrevious((Weight, TaskInQueue)),
-}
 
 pub fn get_transaction_priority_details(tx: &SanitizedTransaction) -> u64 {
         use solana_program_runtime::compute_budget::ComputeBudget;
@@ -1004,7 +998,7 @@ impl ScheduleStage {
         max_executing_queue_count: usize,
         runnable_queue: &mut TaskQueue,
         address_book: &mut AddressBook,
-        from: &crossbeam_channel::Receiver<Multiplexed>,
+        from: &crossbeam_channel::Receiver<TaskInQueue>,
         from_exec: &crossbeam_channel::Receiver<Box<ExecutionEnvironment>>,
         to_execute_substage: &crossbeam_channel::Sender<Box<ExecutionEnvironment>>,
         maybe_to_next_stage: Option<&crossbeam_channel::Sender<Box<ExecutionEnvironment>>>, // assume nonblocking
@@ -1054,13 +1048,8 @@ impl ScheduleStage {
             crossbeam_channel::select! {
                recv(from) -> maybe_from => {
                    trace!("select1: {} {}", from.len(), from_exec.len());
-                   let i = maybe_from.unwrap();
-                    match i {
-                        Multiplexed::FromPrevious(weighted_tx) => {
-                            trace!("recv from previous");
-                            Self::register_runnable_task(weighted_tx, runnable_queue, &mut current_unique_key, &mut sequence_time);
-                        }
-                    }
+                   let task = maybe_from.unwrap();
+                   Self::register_runnable_task(task, runnable_queue, &mut current_unique_key, &mut sequence_time);
                }
                recv(from_exec) -> maybe_from_exec => {
                    trace!("select2: {} {}", from.len(), from_exec.len());

From 505765fe2ef82134730f5947d52d5265f949fecf Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 26 Aug 2022 12:58:27 +0900
Subject: [PATCH 1397/3199] save

---
 scheduler/src/lib.rs | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index ce495ccb07bc6f..720294570fd1b1 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -629,7 +629,7 @@ pub struct ScheduleStage {}
 
 impl ScheduleStage {
     fn push_to_runnable_queue(
-        (weight, mut task): (Weight, TaskInQueue),
+        task: TaskInQueue,
         runnable_queue: &mut TaskQueue,
         unique_key: &mut u64,
     ) {
@@ -645,7 +645,7 @@ impl ScheduleStage {
         //    .unwrap();
         //tx.foo();
         //let unique_weight = (weight << 32) | (*unique_key & 0x0000_0000_ffff_ffff);
-        let unique_weight = weight;
+        let unique_weight = task.unique_weight;
 
         runnable_queue.add_to_schedule(
             /*
@@ -984,12 +984,12 @@ impl ScheduleStage {
 
     #[inline(never)]
     fn register_runnable_task(
-        weighted_tx: (Weight, TaskInQueue),
+        weighted_tx: TaskInQueue,
         runnable_queue: &mut TaskQueue,
         unique_key: &mut u64,
         sequence_time: &mut usize,
     ) {
-        weighted_tx.1.record_sequence_time(*sequence_time);
+        weighted_tx.record_sequence_time(*sequence_time);
         *sequence_time = sequence_time.checked_add(1).unwrap();
         Self::push_to_runnable_queue(weighted_tx, runnable_queue, unique_key)
     }

From ea7e5e086c4bec0c45df60a5e9d57a60952a40fb Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 26 Aug 2022 12:59:54 +0900
Subject: [PATCH 1398/3199] save

---
 transaction-scheduler-bench/src/main.rs | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/transaction-scheduler-bench/src/main.rs b/transaction-scheduler-bench/src/main.rs
index 5e092bab3a0e40..ae8d23af413873 100644
--- a/transaction-scheduler-bench/src/main.rs
+++ b/transaction-scheduler-bench/src/main.rs
@@ -83,7 +83,7 @@ type TransactionMessage = PreprocessedTransaction;
 //type CompletedTransactionMessage = solana_scheduler::Multiplexed; // (usize, Box<solana_scheduler::ExecutionEnvironment>); //(usize, TransactionMessage); // thread index and transaction message
 type CompletedTransactionMessage = Box<solana_scheduler::ExecutionEnvironment>;
 type TransactionBatchMessage = Box<solana_scheduler::ExecutionEnvironment>; // Vec<TransactionMessage>;
-type BatchSenderMessage = solana_scheduler::Multiplexed; // Vec<Vec<PreprocessedTransaction>>;
+type BatchSenderMessage = solana_scheduler::TaskInQueue; // Vec<Vec<PreprocessedTransaction>>;
 
 #[derive(Debug, Default)]
 struct TransactionSchedulerBenchMetrics {
@@ -325,7 +325,7 @@ fn execution_worker(
 fn handle_transaction_batch(
     metrics: &TransactionSchedulerBenchMetrics,
     thread_index: usize,
-    completed_transaction_sender: &(Sender<CompletedTransactionMessage>, Sender<solana_scheduler::Multiplexed>),
+    completed_transaction_sender: &(Sender<CompletedTransactionMessage>),
     mut transaction_batch: TransactionBatchMessage,
     execution_per_tx_us: u64,
 ) {
@@ -517,7 +517,7 @@ fn send_packets(
                 //for lock_attempt in t.tx.1.iter() {
                 //    lock_attempt.contended_unique_weights().insert_task(p, solana_scheduler::TaskInQueue::clone(&t));
                 //}
-                packet_batch_sender.send(solana_scheduler::Multiplexed::FromPrevious((p, t))).unwrap();
+                packet_batch_sender.send(t).unwrap();
             }
         }
         

From f19885723e497db738e1b2bd3e7ceaa462d97454 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 26 Aug 2022 13:00:06 +0900
Subject: [PATCH 1399/3199] save

---
 transaction-scheduler-bench/src/main.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/transaction-scheduler-bench/src/main.rs b/transaction-scheduler-bench/src/main.rs
index ae8d23af413873..9c0ee6ae6d51a8 100644
--- a/transaction-scheduler-bench/src/main.rs
+++ b/transaction-scheduler-bench/src/main.rs
@@ -325,7 +325,7 @@ fn execution_worker(
 fn handle_transaction_batch(
     metrics: &TransactionSchedulerBenchMetrics,
     thread_index: usize,
-    completed_transaction_sender: &(Sender<CompletedTransactionMessage>),
+    completed_transaction_sender: &Sender<CompletedTransactionMessage>,
     mut transaction_batch: TransactionBatchMessage,
     execution_per_tx_us: u64,
 ) {
@@ -398,7 +398,7 @@ fn handle_transaction_batch(
         });
     }
     //error!("send from execute substage: {:?} seq: {}", uq, transaction_batch.task.sequence_time());
-    completed_transaction_sender.0
+    completed_transaction_sender
         .send(transaction_batch)
         .unwrap();
 }

From 746e4992c039b9ec00bf1f328f3c48ca07757608 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 26 Aug 2022 13:00:28 +0900
Subject: [PATCH 1400/3199] save

---
 transaction-scheduler-bench/src/main.rs | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/transaction-scheduler-bench/src/main.rs b/transaction-scheduler-bench/src/main.rs
index 9c0ee6ae6d51a8..1ff7130de0a4cd 100644
--- a/transaction-scheduler-bench/src/main.rs
+++ b/transaction-scheduler-bench/src/main.rs
@@ -258,7 +258,7 @@ fn main() {
 fn start_execution_threads(
     metrics: Arc<TransactionSchedulerBenchMetrics>,
     transaction_batch_receivers: Vec<Receiver<TransactionBatchMessage>>,
-    completed_transaction_sender: (Sender<CompletedTransactionMessage>, Sender<solana_scheduler::Multiplexed>),
+    completed_transaction_sender: Sender<CompletedTransactionMessage>
     execution_per_tx_us: u64,
     exit: Arc<AtomicBool>,
 ) -> Vec<JoinHandle<()>> {
@@ -282,7 +282,7 @@ fn start_execution_thread(
     metrics: Arc<TransactionSchedulerBenchMetrics>,
     thread_index: usize,
     transaction_batch_receiver: Receiver<TransactionBatchMessage>,
-    completed_transaction_sender: (Sender<CompletedTransactionMessage>, Sender<solana_scheduler::Multiplexed>),
+    completed_transaction_sender: Sender<CompletedTransactionMessage>,
     execution_per_tx_us: u64,
     exit: Arc<AtomicBool>,
 ) -> JoinHandle<()> {
@@ -302,7 +302,7 @@ fn execution_worker(
     metrics: Arc<TransactionSchedulerBenchMetrics>,
     thread_index: usize,
     transaction_batch_receiver: Receiver<TransactionBatchMessage>,
-    completed_transaction_sender: (Sender<CompletedTransactionMessage>, Sender<solana_scheduler::Multiplexed>),
+    completed_transaction_sender: Sender<CompletedTransactionMessage>,
     execution_per_tx_us: u64,
     exit: Arc<AtomicBool>,
 ) {

From 7879373afa6ef02fe22bd61f344fd0a350b0a594 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 26 Aug 2022 13:00:37 +0900
Subject: [PATCH 1401/3199] save

---
 transaction-scheduler-bench/src/main.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/transaction-scheduler-bench/src/main.rs b/transaction-scheduler-bench/src/main.rs
index 1ff7130de0a4cd..fb9b18baa75121 100644
--- a/transaction-scheduler-bench/src/main.rs
+++ b/transaction-scheduler-bench/src/main.rs
@@ -258,7 +258,7 @@ fn main() {
 fn start_execution_threads(
     metrics: Arc<TransactionSchedulerBenchMetrics>,
     transaction_batch_receivers: Vec<Receiver<TransactionBatchMessage>>,
-    completed_transaction_sender: Sender<CompletedTransactionMessage>
+    completed_transaction_sender: Sender<CompletedTransactionMessage>,
     execution_per_tx_us: u64,
     exit: Arc<AtomicBool>,
 ) -> Vec<JoinHandle<()>> {

From 0bf499a484c31341824b7cb7fb0008f14c13d144 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 26 Aug 2022 13:00:49 +0900
Subject: [PATCH 1402/3199] save

---
 transaction-scheduler-bench/src/main.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/transaction-scheduler-bench/src/main.rs b/transaction-scheduler-bench/src/main.rs
index fb9b18baa75121..3bec307646f40f 100644
--- a/transaction-scheduler-bench/src/main.rs
+++ b/transaction-scheduler-bench/src/main.rs
@@ -193,7 +193,7 @@ fn main() {
     let execution_handles = start_execution_threads(
         metrics.clone(),
         transaction_batch_receivers,
-        (completed_transaction_sender, packet_batch_sender.clone()),
+        completed_transaction_sender,
         execution_per_tx_us,
         exit.clone(),
     );

From 971a57afeec745c9fc59d79038cdc7d677c23f5e Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 26 Aug 2022 13:01:47 +0900
Subject: [PATCH 1403/3199] save

---
 scheduler/src/lib.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 720294570fd1b1..21d72f3a4f48cd 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -422,7 +422,7 @@ struct Bundle {
 #[derive(Debug)]
 pub struct Task {
     unique_weight: UniqueWeight,
-    pub tx: Box<(SanitizedTransaction, Vec<LockAttempt>)>, // actually should be Bundle
+    pub tx: (SanitizedTransaction, Vec<LockAttempt>), // actually should be Bundle
     pub contention_count: usize,
     pub uncontended: std::sync::atomic::AtomicUsize,
     pub sequence_time: std::sync::atomic::AtomicUsize,
@@ -440,7 +440,7 @@ pub struct Task {
 // commit_time  -+
 
 impl Task {
-    pub fn new_for_queue(unique_weight: UniqueWeight, tx: Box<(SanitizedTransaction, Vec<LockAttempt>)>) -> std::sync::Arc<Self> {
+    pub fn new_for_queue(unique_weight: UniqueWeight, tx: (SanitizedTransaction, Vec<LockAttempt>)) -> std::sync::Arc<Self> {
         TaskInQueue::new(Self { for_indexer: tx.1.iter().map(|a| a.clone_for_test()).collect(), unique_weight, tx, contention_count: 0, uncontended: Default::default(), sequence_time: std::sync::atomic::AtomicUsize::new(usize::max_value()), sequence_end_time: std::sync::atomic::AtomicUsize::new(usize::max_value()), queue_time: std::sync::atomic::AtomicUsize::new(usize::max_value()), queue_end_time: std::sync::atomic::AtomicUsize::new(usize::max_value()), execute_time: std::sync::atomic::AtomicUsize::new(usize::max_value()), commit_time: std::sync::atomic::AtomicUsize::new(usize::max_value()),  })
     }
 

From d83d7c71e53b0841c85172d1d74aaa03e243b1a5 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 26 Aug 2022 13:01:58 +0900
Subject: [PATCH 1404/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 21d72f3a4f48cd..464f87107865a6 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -508,7 +508,7 @@ impl Task {
         Self {
             unique_weight: self.unique_weight,
             for_indexer: self.tx.1.iter().map(|a| a.clone_for_test()).collect(),
-            tx: Box::new((self.tx.0.clone(), self.tx.1.iter().map(|l| l.clone_for_test()).collect::<Vec<_>>())),
+            tx: (self.tx.0.clone(), self.tx.1.iter().map(|l| l.clone_for_test()).collect::<Vec<_>>()),
             contention_count: Default::default(),
             uncontended: Default::default(),
             sequence_time: std::sync::atomic::AtomicUsize::new(usize::max_value()),

From a7a71f0362b47598addc8fce4b1742b93923d3a4 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 26 Aug 2022 13:02:43 +0900
Subject: [PATCH 1405/3199] save

---
 transaction-scheduler-bench/src/main.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/transaction-scheduler-bench/src/main.rs b/transaction-scheduler-bench/src/main.rs
index 3bec307646f40f..c002ade1bcea82 100644
--- a/transaction-scheduler-bench/src/main.rs
+++ b/transaction-scheduler-bench/src/main.rs
@@ -78,7 +78,7 @@ struct Args {
 }
 
 /// Some convenient type aliases
-type PreprocessedTransaction = Box<(SanitizedTransaction, Vec<solana_scheduler::LockAttempt>)>;
+type PreprocessedTransaction = (SanitizedTransaction, Vec<solana_scheduler::LockAttempt>);
 type TransactionMessage = PreprocessedTransaction;
 //type CompletedTransactionMessage = solana_scheduler::Multiplexed; // (usize, Box<solana_scheduler::ExecutionEnvironment>); //(usize, TransactionMessage); // thread index and transaction message
 type CompletedTransactionMessage = Box<solana_scheduler::ExecutionEnvironment>;
@@ -593,7 +593,7 @@ fn build_packet(
         .map(|address| solana_scheduler::LockAttempt::new(preloader.load(**address), solana_scheduler::RequestedUsage::Readonly));
     let locks = writable_lock_iter.chain(readonly_lock_iter).collect::<Vec<_>>();
 
-    Box::new((sanitized_tx, locks))
+    (sanitized_tx, locks)
 }
 
 fn build_accounts(num_accounts: usize) -> Vec<Keypair> {

From 2a0a561c376d76569aa222bf1217581c2abee33f Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 26 Aug 2022 13:04:19 +0900
Subject: [PATCH 1406/3199] save

---
 scheduler/src/lib.rs                    | 6 +++---
 transaction-scheduler-bench/src/main.rs | 4 ++--
 2 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 464f87107865a6..720294570fd1b1 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -422,7 +422,7 @@ struct Bundle {
 #[derive(Debug)]
 pub struct Task {
     unique_weight: UniqueWeight,
-    pub tx: (SanitizedTransaction, Vec<LockAttempt>), // actually should be Bundle
+    pub tx: Box<(SanitizedTransaction, Vec<LockAttempt>)>, // actually should be Bundle
     pub contention_count: usize,
     pub uncontended: std::sync::atomic::AtomicUsize,
     pub sequence_time: std::sync::atomic::AtomicUsize,
@@ -440,7 +440,7 @@ pub struct Task {
 // commit_time  -+
 
 impl Task {
-    pub fn new_for_queue(unique_weight: UniqueWeight, tx: (SanitizedTransaction, Vec<LockAttempt>)) -> std::sync::Arc<Self> {
+    pub fn new_for_queue(unique_weight: UniqueWeight, tx: Box<(SanitizedTransaction, Vec<LockAttempt>)>) -> std::sync::Arc<Self> {
         TaskInQueue::new(Self { for_indexer: tx.1.iter().map(|a| a.clone_for_test()).collect(), unique_weight, tx, contention_count: 0, uncontended: Default::default(), sequence_time: std::sync::atomic::AtomicUsize::new(usize::max_value()), sequence_end_time: std::sync::atomic::AtomicUsize::new(usize::max_value()), queue_time: std::sync::atomic::AtomicUsize::new(usize::max_value()), queue_end_time: std::sync::atomic::AtomicUsize::new(usize::max_value()), execute_time: std::sync::atomic::AtomicUsize::new(usize::max_value()), commit_time: std::sync::atomic::AtomicUsize::new(usize::max_value()),  })
     }
 
@@ -508,7 +508,7 @@ impl Task {
         Self {
             unique_weight: self.unique_weight,
             for_indexer: self.tx.1.iter().map(|a| a.clone_for_test()).collect(),
-            tx: (self.tx.0.clone(), self.tx.1.iter().map(|l| l.clone_for_test()).collect::<Vec<_>>()),
+            tx: Box::new((self.tx.0.clone(), self.tx.1.iter().map(|l| l.clone_for_test()).collect::<Vec<_>>())),
             contention_count: Default::default(),
             uncontended: Default::default(),
             sequence_time: std::sync::atomic::AtomicUsize::new(usize::max_value()),
diff --git a/transaction-scheduler-bench/src/main.rs b/transaction-scheduler-bench/src/main.rs
index c002ade1bcea82..3bec307646f40f 100644
--- a/transaction-scheduler-bench/src/main.rs
+++ b/transaction-scheduler-bench/src/main.rs
@@ -78,7 +78,7 @@ struct Args {
 }
 
 /// Some convenient type aliases
-type PreprocessedTransaction = (SanitizedTransaction, Vec<solana_scheduler::LockAttempt>);
+type PreprocessedTransaction = Box<(SanitizedTransaction, Vec<solana_scheduler::LockAttempt>)>;
 type TransactionMessage = PreprocessedTransaction;
 //type CompletedTransactionMessage = solana_scheduler::Multiplexed; // (usize, Box<solana_scheduler::ExecutionEnvironment>); //(usize, TransactionMessage); // thread index and transaction message
 type CompletedTransactionMessage = Box<solana_scheduler::ExecutionEnvironment>;
@@ -593,7 +593,7 @@ fn build_packet(
         .map(|address| solana_scheduler::LockAttempt::new(preloader.load(**address), solana_scheduler::RequestedUsage::Readonly));
     let locks = writable_lock_iter.chain(readonly_lock_iter).collect::<Vec<_>>();
 
-    (sanitized_tx, locks)
+    Box::new((sanitized_tx, locks))
 }
 
 fn build_accounts(num_accounts: usize) -> Vec<Keypair> {

From f7abeb2b985f43321c530687d3c468bcd3b75eeb Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 26 Aug 2022 13:05:26 +0900
Subject: [PATCH 1407/3199] save

---
 scheduler/src/lib.rs                    | 6 +++---
 transaction-scheduler-bench/src/main.rs | 4 ++--
 2 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 720294570fd1b1..464f87107865a6 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -422,7 +422,7 @@ struct Bundle {
 #[derive(Debug)]
 pub struct Task {
     unique_weight: UniqueWeight,
-    pub tx: Box<(SanitizedTransaction, Vec<LockAttempt>)>, // actually should be Bundle
+    pub tx: (SanitizedTransaction, Vec<LockAttempt>), // actually should be Bundle
     pub contention_count: usize,
     pub uncontended: std::sync::atomic::AtomicUsize,
     pub sequence_time: std::sync::atomic::AtomicUsize,
@@ -440,7 +440,7 @@ pub struct Task {
 // commit_time  -+
 
 impl Task {
-    pub fn new_for_queue(unique_weight: UniqueWeight, tx: Box<(SanitizedTransaction, Vec<LockAttempt>)>) -> std::sync::Arc<Self> {
+    pub fn new_for_queue(unique_weight: UniqueWeight, tx: (SanitizedTransaction, Vec<LockAttempt>)) -> std::sync::Arc<Self> {
         TaskInQueue::new(Self { for_indexer: tx.1.iter().map(|a| a.clone_for_test()).collect(), unique_weight, tx, contention_count: 0, uncontended: Default::default(), sequence_time: std::sync::atomic::AtomicUsize::new(usize::max_value()), sequence_end_time: std::sync::atomic::AtomicUsize::new(usize::max_value()), queue_time: std::sync::atomic::AtomicUsize::new(usize::max_value()), queue_end_time: std::sync::atomic::AtomicUsize::new(usize::max_value()), execute_time: std::sync::atomic::AtomicUsize::new(usize::max_value()), commit_time: std::sync::atomic::AtomicUsize::new(usize::max_value()),  })
     }
 
@@ -508,7 +508,7 @@ impl Task {
         Self {
             unique_weight: self.unique_weight,
             for_indexer: self.tx.1.iter().map(|a| a.clone_for_test()).collect(),
-            tx: Box::new((self.tx.0.clone(), self.tx.1.iter().map(|l| l.clone_for_test()).collect::<Vec<_>>())),
+            tx: (self.tx.0.clone(), self.tx.1.iter().map(|l| l.clone_for_test()).collect::<Vec<_>>()),
             contention_count: Default::default(),
             uncontended: Default::default(),
             sequence_time: std::sync::atomic::AtomicUsize::new(usize::max_value()),
diff --git a/transaction-scheduler-bench/src/main.rs b/transaction-scheduler-bench/src/main.rs
index 3bec307646f40f..c002ade1bcea82 100644
--- a/transaction-scheduler-bench/src/main.rs
+++ b/transaction-scheduler-bench/src/main.rs
@@ -78,7 +78,7 @@ struct Args {
 }
 
 /// Some convenient type aliases
-type PreprocessedTransaction = Box<(SanitizedTransaction, Vec<solana_scheduler::LockAttempt>)>;
+type PreprocessedTransaction = (SanitizedTransaction, Vec<solana_scheduler::LockAttempt>);
 type TransactionMessage = PreprocessedTransaction;
 //type CompletedTransactionMessage = solana_scheduler::Multiplexed; // (usize, Box<solana_scheduler::ExecutionEnvironment>); //(usize, TransactionMessage); // thread index and transaction message
 type CompletedTransactionMessage = Box<solana_scheduler::ExecutionEnvironment>;
@@ -593,7 +593,7 @@ fn build_packet(
         .map(|address| solana_scheduler::LockAttempt::new(preloader.load(**address), solana_scheduler::RequestedUsage::Readonly));
     let locks = writable_lock_iter.chain(readonly_lock_iter).collect::<Vec<_>>();
 
-    Box::new((sanitized_tx, locks))
+    (sanitized_tx, locks)
 }
 
 fn build_accounts(num_accounts: usize) -> Vec<Keypair> {

From 0c370be9d453c0d5ffc7149d2bd44ec1f4f7c264 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 26 Aug 2022 13:13:01 +0900
Subject: [PATCH 1408/3199] save

---
 scheduler/src/lib.rs | 51 ++++++++++++++++++++++++++++++++------------
 1 file changed, 37 insertions(+), 14 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 464f87107865a6..0160e356761f3e 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1047,12 +1047,12 @@ impl ScheduleStage {
 
             crossbeam_channel::select! {
                recv(from) -> maybe_from => {
-                   trace!("select1: {} {}", from.len(), from_exec.len());
+                   //trace!("select1: {} {}", from.len(), from_exec.len());
                    let task = maybe_from.unwrap();
                    Self::register_runnable_task(task, runnable_queue, &mut current_unique_key, &mut sequence_time);
                }
                recv(from_exec) -> maybe_from_exec => {
-                   trace!("select2: {} {}", from.len(), from_exec.len());
+                   //trace!("select2: {} {}", from.len(), from_exec.len());
                    let mut processed_execution_environment = maybe_from_exec.unwrap();
                     trace!("recv from execute: {:?}", processed_execution_environment.unique_weight);
                     executing_queue_count -= 1;
@@ -1064,22 +1064,45 @@ impl ScheduleStage {
             }
 
             loop {
-                if (executing_queue_count + address_book.provisioning_trackers.len()) >= max_executing_queue_count {
-                    trace!("skip scheduling; outgoing queue full");
-                    break;
-                }
+                loop {
+                    if (executing_queue_count + address_book.provisioning_trackers.len()) >= max_executing_queue_count {
+                        trace!("skip scheduling; outgoing queue full");
+                        break;
+                    }
 
-                let prefer_immediate = address_book.provisioning_trackers.len()/4 > executing_queue_count;
-                let maybe_ee =
-                    Self::schedule_next_execution(&task_sender, runnable_queue, address_book, &mut contended_count, prefer_immediate, &sequence_time, &mut queue_clock, &mut execute_clock);
+                    let prefer_immediate = address_book.provisioning_trackers.len()/4 > executing_queue_count;
+                    let maybe_ee =
+                        Self::schedule_next_execution(&task_sender, runnable_queue, address_book, &mut contended_count, prefer_immediate, &sequence_time, &mut queue_clock, &mut execute_clock);
 
-                if let Some(ee) = maybe_ee {
-                    trace!("send to execute");
-                    executing_queue_count += 1;
+                    if let Some(ee) = maybe_ee {
+                        trace!("send to execute");
+                        executing_queue_count += 1;
 
-                    to_execute_substage.send(ee).unwrap();
-                } else {
+                        to_execute_substage.send(ee).unwrap();
+                    } else {
+                        break;
+                    }
+                }
+
+                let from_len = from.len();
+                let from_exec_len v from_exec.len();
+
+                if from_len == 0 && from_exec_len == 0 {
                     break;
+                } else {
+                    if from_len > 0 {
+                       let task = from.recv().unwrap();
+                       Self::register_runnable_task(task, runnable_queue, &mut current_unique_key, &mut sequence_time);
+                    }
+                    if from_exec_len > 0 {
+                        let mut processed_execution_environment = from_exec.recv().unwrap();
+                        trace!("recv from execute: {:?}", processed_execution_environment.unique_weight);
+                        executing_queue_count -= 1;
+
+                        Self::commit_completed_execution(&mut processed_execution_environment, address_book, &mut execute_clock);
+                        // async-ly propagate the result to rpc subsystems
+                        to_next_stage.send(processed_execution_environment).unwrap();
+                    }
                 }
             }
         }

From 233993d83148d5b892a1fb1dff16bb5a3a91c72e Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 26 Aug 2022 13:13:20 +0900
Subject: [PATCH 1409/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 0160e356761f3e..2e29ae34977a6e 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1085,7 +1085,7 @@ impl ScheduleStage {
                 }
 
                 let from_len = from.len();
-                let from_exec_len v from_exec.len();
+                let from_exec_len = from_exec.len();
 
                 if from_len == 0 && from_exec_len == 0 {
                     break;

From 812c772bad40e0e874a89ceeaefee79d691fa6d1 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 26 Aug 2022 13:15:02 +0900
Subject: [PATCH 1410/3199] save

---
 scheduler/src/lib.rs | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 2e29ae34977a6e..3e1eb4c5480ed8 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1047,12 +1047,12 @@ impl ScheduleStage {
 
             crossbeam_channel::select! {
                recv(from) -> maybe_from => {
-                   //trace!("select1: {} {}", from.len(), from_exec.len());
+                   trace!("select1: {} {}", from.len(), from_exec.len());
                    let task = maybe_from.unwrap();
                    Self::register_runnable_task(task, runnable_queue, &mut current_unique_key, &mut sequence_time);
                }
                recv(from_exec) -> maybe_from_exec => {
-                   //trace!("select2: {} {}", from.len(), from_exec.len());
+                   trace!("select2: {} {}", from.len(), from_exec.len());
                    let mut processed_execution_environment = maybe_from_exec.unwrap();
                     trace!("recv from execute: {:?}", processed_execution_environment.unique_weight);
                     executing_queue_count -= 1;
@@ -1088,6 +1088,7 @@ impl ScheduleStage {
                 let from_exec_len = from_exec.len();
 
                 if from_len == 0 && from_exec_len == 0 {
+                   trace!("select: back to");
                     break;
                 } else {
                     if from_len > 0 {

From 93da1d4a026049a1c25a5bbb4cc85933302387d2 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 26 Aug 2022 13:18:47 +0900
Subject: [PATCH 1411/3199] save

---
 scheduler/src/lib.rs | 25 +++++++++++++++++++------
 1 file changed, 19 insertions(+), 6 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 3e1eb4c5480ed8..498a56e9884f48 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1042,17 +1042,20 @@ impl ScheduleStage {
             }).unwrap();
         }
 
+        let mut from_len = from.len();
+        let mut from_exec_len = from_exec.len();
+
         loop {
-            trace!("schedule_once (from: {}, to: {}, runnnable: {}, contended: {}, (immediate+provisional)/max: ({}+{})/{}) active from contended: {}!", from.len(), to_execute_substage.len(), runnable_queue.task_count(), contended_count, executing_queue_count, address_book.provisioning_trackers.len(), max_executing_queue_count, address_book.uncontended_task_ids.len());
+            trace!("schedule_once (from: {}, to: {}, runnnable: {}, contended: {}, (immediate+provisional)/max: ({}+{})/{}) active from contended: {}!", from_len, to_execute_substage.len(), runnable_queue.task_count(), contended_count, executing_queue_count, address_book.provisioning_trackers.len(), max_executing_queue_count, address_book.uncontended_task_ids.len());
 
             crossbeam_channel::select! {
                recv(from) -> maybe_from => {
-                   trace!("select1: {} {}", from.len(), from_exec.len());
+                   trace!("select1: {} {}", from_len, from_exec_len);
                    let task = maybe_from.unwrap();
                    Self::register_runnable_task(task, runnable_queue, &mut current_unique_key, &mut sequence_time);
                }
                recv(from_exec) -> maybe_from_exec => {
-                   trace!("select2: {} {}", from.len(), from_exec.len());
+                   trace!("select2: {} {}", from_len, from_exec_len);
                    let mut processed_execution_environment = maybe_from_exec.unwrap();
                     trace!("recv from execute: {:?}", processed_execution_environment.unique_weight);
                     executing_queue_count -= 1;
@@ -1084,18 +1087,28 @@ impl ScheduleStage {
                     }
                 }
 
-                let from_len = from.len();
-                let from_exec_len = from_exec.len();
+                from_len = from.len();
+                from_exec_len = from_exec.len();
 
                 if from_len == 0 && from_exec_len == 0 {
                    trace!("select: back to");
-                    break;
+                   break;
                 } else {
                     if from_len > 0 {
+                       from_len -= 1;
+                       if from_len == 0 {
+                           from_len = from.len();
+                       }
+                       trace!("select3: {} {}", from_len, from_exec_len);
                        let task = from.recv().unwrap();
                        Self::register_runnable_task(task, runnable_queue, &mut current_unique_key, &mut sequence_time);
                     }
                     if from_exec_len > 0 {
+                       from_exec_len -= 1;
+                       if from_exec_len == 0 {
+                           from_exec_len = from_exec.len();
+                       }
+                       trace!("select4: {} {}", from_len, from_exec_len);
                         let mut processed_execution_environment = from_exec.recv().unwrap();
                         trace!("recv from execute: {:?}", processed_execution_environment.unique_weight);
                         executing_queue_count -= 1;

From ae3eca457a08efafe76100623762be419f8b3941 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 26 Aug 2022 13:19:52 +0900
Subject: [PATCH 1412/3199] save

---
 scheduler/src/lib.rs | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 498a56e9884f48..7a902cf17a6fd6 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1095,19 +1095,15 @@ impl ScheduleStage {
                    break;
                 } else {
                     if from_len > 0 {
+                       trace!("select3: {} {}", from_len, from_exec_len);
+                       let task = from.recv().unwrap();
+                       Self::register_runnable_task(task, runnable_queue, &mut current_unique_key, &mut sequence_time);
                        from_len -= 1;
                        if from_len == 0 {
                            from_len = from.len();
                        }
-                       trace!("select3: {} {}", from_len, from_exec_len);
-                       let task = from.recv().unwrap();
-                       Self::register_runnable_task(task, runnable_queue, &mut current_unique_key, &mut sequence_time);
                     }
                     if from_exec_len > 0 {
-                       from_exec_len -= 1;
-                       if from_exec_len == 0 {
-                           from_exec_len = from_exec.len();
-                       }
                        trace!("select4: {} {}", from_len, from_exec_len);
                         let mut processed_execution_environment = from_exec.recv().unwrap();
                         trace!("recv from execute: {:?}", processed_execution_environment.unique_weight);
@@ -1116,6 +1112,10 @@ impl ScheduleStage {
                         Self::commit_completed_execution(&mut processed_execution_environment, address_book, &mut execute_clock);
                         // async-ly propagate the result to rpc subsystems
                         to_next_stage.send(processed_execution_environment).unwrap();
+                       from_exec_len -= 1;
+                       if from_exec_len == 0 {
+                           from_exec_len = from_exec.len();
+                       }
                     }
                 }
             }

From 6284926204eb6964d8ecf64e6ca59be9233922a2 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 26 Aug 2022 13:25:36 +0900
Subject: [PATCH 1413/3199] save

---
 scheduler/src/lib.rs | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 7a902cf17a6fd6..f0516c89c9266f 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1046,16 +1046,16 @@ impl ScheduleStage {
         let mut from_exec_len = from_exec.len();
 
         loop {
-            trace!("schedule_once (from: {}, to: {}, runnnable: {}, contended: {}, (immediate+provisional)/max: ({}+{})/{}) active from contended: {}!", from_len, to_execute_substage.len(), runnable_queue.task_count(), contended_count, executing_queue_count, address_book.provisioning_trackers.len(), max_executing_queue_count, address_book.uncontended_task_ids.len());
+            //trace!("schedule_once (from: {}, to: {}, runnnable: {}, contended: {}, (immediate+provisional)/max: ({}+{})/{}) active from contended: {}!", from_len, to_execute_substage.len(), runnable_queue.task_count(), contended_count, executing_queue_count, address_book.provisioning_trackers.len(), max_executing_queue_count, address_book.uncontended_task_ids.len());
 
             crossbeam_channel::select! {
                recv(from) -> maybe_from => {
-                   trace!("select1: {} {}", from_len, from_exec_len);
+                   trace!("select1: {} {}", from.len(), from_exec.len());
                    let task = maybe_from.unwrap();
                    Self::register_runnable_task(task, runnable_queue, &mut current_unique_key, &mut sequence_time);
                }
                recv(from_exec) -> maybe_from_exec => {
-                   trace!("select2: {} {}", from_len, from_exec_len);
+                   trace!("select2: {} {}", from.len(), from_exec.len());
                    let mut processed_execution_environment = maybe_from_exec.unwrap();
                     trace!("recv from execute: {:?}", processed_execution_environment.unique_weight);
                     executing_queue_count -= 1;

From 217bc42e5479040efcb19d511e2a94ada9fc338f Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 26 Aug 2022 13:27:16 +0900
Subject: [PATCH 1414/3199] save

---
 scheduler/src/lib.rs | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index f0516c89c9266f..914d228c3aa561 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1050,14 +1050,14 @@ impl ScheduleStage {
 
             crossbeam_channel::select! {
                recv(from) -> maybe_from => {
-                   trace!("select1: {} {}", from.len(), from_exec.len());
+                   //trace!("select1: {} {}", from.len(), from_exec.len());
                    let task = maybe_from.unwrap();
                    Self::register_runnable_task(task, runnable_queue, &mut current_unique_key, &mut sequence_time);
                }
                recv(from_exec) -> maybe_from_exec => {
-                   trace!("select2: {} {}", from.len(), from_exec.len());
+                   //trace!("select2: {} {}", from.len(), from_exec.len());
                    let mut processed_execution_environment = maybe_from_exec.unwrap();
-                    trace!("recv from execute: {:?}", processed_execution_environment.unique_weight);
+                    //trace!("recv from execute: {:?}", processed_execution_environment.unique_weight);
                     executing_queue_count -= 1;
 
                     Self::commit_completed_execution(&mut processed_execution_environment, address_book, &mut execute_clock);
@@ -1069,7 +1069,7 @@ impl ScheduleStage {
             loop {
                 loop {
                     if (executing_queue_count + address_book.provisioning_trackers.len()) >= max_executing_queue_count {
-                        trace!("skip scheduling; outgoing queue full");
+                        //trace!("skip scheduling; outgoing queue full");
                         break;
                     }
 
@@ -1078,7 +1078,7 @@ impl ScheduleStage {
                         Self::schedule_next_execution(&task_sender, runnable_queue, address_book, &mut contended_count, prefer_immediate, &sequence_time, &mut queue_clock, &mut execute_clock);
 
                     if let Some(ee) = maybe_ee {
-                        trace!("send to execute");
+                        //trace!("send to execute");
                         executing_queue_count += 1;
 
                         to_execute_substage.send(ee).unwrap();
@@ -1091,7 +1091,7 @@ impl ScheduleStage {
                 from_exec_len = from_exec.len();
 
                 if from_len == 0 && from_exec_len == 0 {
-                   trace!("select: back to");
+                   //trace!("select: back to");
                    break;
                 } else {
                     if from_len > 0 {

From f0c6156582c2540923b53d005b495ccf9ca73494 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 26 Aug 2022 13:28:15 +0900
Subject: [PATCH 1415/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 914d228c3aa561..cae5c245b409e2 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1091,7 +1091,7 @@ impl ScheduleStage {
                 from_exec_len = from_exec.len();
 
                 if from_len == 0 && from_exec_len == 0 {
-                   //trace!("select: back to");
+                   trace!("select: back to");
                    break;
                 } else {
                     if from_len > 0 {

From 662a2eff48929813714cfb37c9ec6c35819f4a10 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 26 Aug 2022 13:29:28 +0900
Subject: [PATCH 1416/3199] save

---
 scheduler/src/lib.rs | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index cae5c245b409e2..e55a4f66824c4e 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1101,6 +1101,7 @@ impl ScheduleStage {
                        from_len -= 1;
                        if from_len == 0 {
                            from_len = from.len();
+                           trace!("select3: refill {} {}", from_len, from_exec_len);
                        }
                     }
                     if from_exec_len > 0 {
@@ -1115,6 +1116,7 @@ impl ScheduleStage {
                        from_exec_len -= 1;
                        if from_exec_len == 0 {
                            from_exec_len = from_exec.len();
+                           trace!("select4: refill {} {}", from_len, from_exec_len);
                        }
                     }
                 }

From fdf3193f16f6c53633429153c5126e024fd15231 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 26 Aug 2022 13:30:37 +0900
Subject: [PATCH 1417/3199] save

---
 scheduler/src/lib.rs | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index e55a4f66824c4e..42d8611e6a3e9d 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1114,8 +1114,7 @@ impl ScheduleStage {
                         // async-ly propagate the result to rpc subsystems
                         to_next_stage.send(processed_execution_environment).unwrap();
                        from_exec_len -= 1;
-                       if from_exec_len == 0 {
-                           from_exec_len = from_exec.len();
+                       if from_exec_len == 0 && (from_exec_len = from_exec.len()) > 0 {
                            trace!("select4: refill {} {}", from_len, from_exec_len);
                        }
                     }

From 5cba82a1afbc91c82b7054f681079726cc63abfc Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 26 Aug 2022 13:30:56 +0900
Subject: [PATCH 1418/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 42d8611e6a3e9d..7a5e40c600b1bf 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1114,7 +1114,7 @@ impl ScheduleStage {
                         // async-ly propagate the result to rpc subsystems
                         to_next_stage.send(processed_execution_environment).unwrap();
                        from_exec_len -= 1;
-                       if from_exec_len == 0 && (from_exec_len = from_exec.len()) > 0 {
+                       if from_exec_len == 0 && {from_exec_len = from_exec.len()} > 0 {
                            trace!("select4: refill {} {}", from_len, from_exec_len);
                        }
                     }

From 004bb2a0bdedb86337986aa0a2106723b275c1e1 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 26 Aug 2022 13:31:42 +0900
Subject: [PATCH 1419/3199] save

---
 scheduler/src/lib.rs | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 7a5e40c600b1bf..13113152435d33 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1114,7 +1114,8 @@ impl ScheduleStage {
                         // async-ly propagate the result to rpc subsystems
                         to_next_stage.send(processed_execution_environment).unwrap();
                        from_exec_len -= 1;
-                       if from_exec_len == 0 && {from_exec_len = from_exec.len()} > 0 {
+                       if from_exec_len == 0 && {
+                           from_exec_len = from_exec.len();
                            trace!("select4: refill {} {}", from_len, from_exec_len);
                        }
                     }

From ba88b382541818c345b99fdf0e9b9a44807f389e Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 26 Aug 2022 13:32:14 +0900
Subject: [PATCH 1420/3199] save

---
 scheduler/src/lib.rs | 10 +++++++---
 1 file changed, 7 insertions(+), 3 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 13113152435d33..3f9fea0068b5b2 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1101,7 +1101,9 @@ impl ScheduleStage {
                        from_len -= 1;
                        if from_len == 0 {
                            from_len = from.len();
-                           trace!("select3: refill {} {}", from_len, from_exec_len);
+                           if from_len > 0 {
+                               trace!("select3: refill {} {}", from_len, from_exec_len);
+                           }
                        }
                     }
                     if from_exec_len > 0 {
@@ -1114,9 +1116,11 @@ impl ScheduleStage {
                         // async-ly propagate the result to rpc subsystems
                         to_next_stage.send(processed_execution_environment).unwrap();
                        from_exec_len -= 1;
-                       if from_exec_len == 0 && {
+                       if from_exec_len == 0 {
                            from_exec_len = from_exec.len();
-                           trace!("select4: refill {} {}", from_len, from_exec_len);
+                           if from_exec_len > 0 {
+                               trace!("select3: refill {} {}", from_len, from_exec_len);
+                           }
                        }
                     }
                 }

From 276a3ced3f494ee4e9d2ae5b3760ca935cb09e40 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 26 Aug 2022 13:39:27 +0900
Subject: [PATCH 1421/3199] save

---
 scheduler/src/lib.rs | 25 ++++++++++++++++++++++---
 1 file changed, 22 insertions(+), 3 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 3f9fea0068b5b2..1a7f3ebf0c8a67 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1066,7 +1066,6 @@ impl ScheduleStage {
                }
             }
 
-            loop {
                 loop {
                     if (executing_queue_count + address_book.provisioning_trackers.len()) >= max_executing_queue_count {
                         //trace!("skip scheduling; outgoing queue full");
@@ -1087,9 +1086,10 @@ impl ScheduleStage {
                     }
                 }
 
-                from_len = from.len();
-                from_exec_len = from_exec.len();
+            from_len = from.len();
+            from_exec_len = from_exec.len();
 
+            loop {
                 if from_len == 0 && from_exec_len == 0 {
                    trace!("select: back to");
                    break;
@@ -1123,6 +1123,25 @@ impl ScheduleStage {
                            }
                        }
                     }
+                    loop {
+                        if (executing_queue_count + address_book.provisioning_trackers.len()) >= max_executing_queue_count {
+                            //trace!("skip scheduling; outgoing queue full");
+                            break;
+                        }
+
+                        let prefer_immediate = address_book.provisioning_trackers.len()/4 > executing_queue_count;
+                        let maybe_ee =
+                            Self::schedule_next_execution(&task_sender, runnable_queue, address_book, &mut contended_count, prefer_immediate, &sequence_time, &mut queue_clock, &mut execute_clock);
+
+                        if let Some(ee) = maybe_ee {
+                            //trace!("send to execute");
+                            executing_queue_count += 1;
+
+                            to_execute_substage.send(ee).unwrap();
+                        } else {
+                            break;
+                        }
+                    }
                 }
             }
         }

From 29b97c364daa18ee7dbc9273d49ec826967ddd51 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 26 Aug 2022 13:40:22 +0900
Subject: [PATCH 1422/3199] save

---
 scheduler/src/lib.rs | 25 +++----------------------
 1 file changed, 3 insertions(+), 22 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 1a7f3ebf0c8a67..3f9fea0068b5b2 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1066,6 +1066,7 @@ impl ScheduleStage {
                }
             }
 
+            loop {
                 loop {
                     if (executing_queue_count + address_book.provisioning_trackers.len()) >= max_executing_queue_count {
                         //trace!("skip scheduling; outgoing queue full");
@@ -1086,10 +1087,9 @@ impl ScheduleStage {
                     }
                 }
 
-            from_len = from.len();
-            from_exec_len = from_exec.len();
+                from_len = from.len();
+                from_exec_len = from_exec.len();
 
-            loop {
                 if from_len == 0 && from_exec_len == 0 {
                    trace!("select: back to");
                    break;
@@ -1123,25 +1123,6 @@ impl ScheduleStage {
                            }
                        }
                     }
-                    loop {
-                        if (executing_queue_count + address_book.provisioning_trackers.len()) >= max_executing_queue_count {
-                            //trace!("skip scheduling; outgoing queue full");
-                            break;
-                        }
-
-                        let prefer_immediate = address_book.provisioning_trackers.len()/4 > executing_queue_count;
-                        let maybe_ee =
-                            Self::schedule_next_execution(&task_sender, runnable_queue, address_book, &mut contended_count, prefer_immediate, &sequence_time, &mut queue_clock, &mut execute_clock);
-
-                        if let Some(ee) = maybe_ee {
-                            //trace!("send to execute");
-                            executing_queue_count += 1;
-
-                            to_execute_substage.send(ee).unwrap();
-                        } else {
-                            break;
-                        }
-                    }
                 }
             }
         }

From 34b519f10bb47bb99f1a8cc381c02389de631a79 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 26 Aug 2022 15:19:31 +0900
Subject: [PATCH 1423/3199] save

---
 scheduler/src/lib.rs | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 3f9fea0068b5b2..b0da5924ae219a 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1066,6 +1066,9 @@ impl ScheduleStage {
                }
             }
 
+            from_len = from.len();
+            from_exec_len = from_exec.len();
+
             loop {
                 loop {
                     if (executing_queue_count + address_book.provisioning_trackers.len()) >= max_executing_queue_count {
@@ -1087,9 +1090,6 @@ impl ScheduleStage {
                     }
                 }
 
-                from_len = from.len();
-                from_exec_len = from_exec.len();
-
                 if from_len == 0 && from_exec_len == 0 {
                    trace!("select: back to");
                    break;

From 3fa0c514520ea14faae62757ffc2f4d9051035f4 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 26 Aug 2022 15:21:29 +0900
Subject: [PATCH 1424/3199] save

---
 scheduler/src/lib.rs | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index b0da5924ae219a..3f9fea0068b5b2 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1066,9 +1066,6 @@ impl ScheduleStage {
                }
             }
 
-            from_len = from.len();
-            from_exec_len = from_exec.len();
-
             loop {
                 loop {
                     if (executing_queue_count + address_book.provisioning_trackers.len()) >= max_executing_queue_count {
@@ -1090,6 +1087,9 @@ impl ScheduleStage {
                     }
                 }
 
+                from_len = from.len();
+                from_exec_len = from_exec.len();
+
                 if from_len == 0 && from_exec_len == 0 {
                    trace!("select: back to");
                    break;

From ac2656dc614ac773f6869b9e7d624347a3b5959b Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 26 Aug 2022 15:22:12 +0900
Subject: [PATCH 1425/3199] save

---
 scheduler/src/lib.rs | 31 ++++++++++++++++---------------
 1 file changed, 16 insertions(+), 15 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 3f9fea0068b5b2..8f62ed77825479 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1067,25 +1067,26 @@ impl ScheduleStage {
             }
 
             loop {
-                loop {
-                    if (executing_queue_count + address_book.provisioning_trackers.len()) >= max_executing_queue_count {
-                        //trace!("skip scheduling; outgoing queue full");
-                        break;
-                    }
+                if (executing_queue_count + address_book.provisioning_trackers.len()) >= max_executing_queue_count {
+                    //trace!("skip scheduling; outgoing queue full");
+                    break;
+                }
 
-                    let prefer_immediate = address_book.provisioning_trackers.len()/4 > executing_queue_count;
-                    let maybe_ee =
-                        Self::schedule_next_execution(&task_sender, runnable_queue, address_book, &mut contended_count, prefer_immediate, &sequence_time, &mut queue_clock, &mut execute_clock);
+                let prefer_immediate = address_book.provisioning_trackers.len()/4 > executing_queue_count;
+                let maybe_ee =
+                    Self::schedule_next_execution(&task_sender, runnable_queue, address_book, &mut contended_count, prefer_immediate, &sequence_time, &mut queue_clock, &mut execute_clock);
 
-                    if let Some(ee) = maybe_ee {
-                        //trace!("send to execute");
-                        executing_queue_count += 1;
+                if let Some(ee) = maybe_ee {
+                    //trace!("send to execute");
+                    executing_queue_count += 1;
 
-                        to_execute_substage.send(ee).unwrap();
-                    } else {
-                        break;
-                    }
+                    to_execute_substage.send(ee).unwrap();
+                } else {
+                    break;
                 }
+            }
+
+            loop {
 
                 from_len = from.len();
                 from_exec_len = from_exec.len();

From 25d8765a62b2b2f0465a25a1bff746fd5d29469c Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 26 Aug 2022 15:23:14 +0900
Subject: [PATCH 1426/3199] save

---
 scheduler/src/lib.rs | 19 +++++++++++++++++++
 1 file changed, 19 insertions(+)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 8f62ed77825479..67fdbc99ac22ed 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1125,6 +1125,25 @@ impl ScheduleStage {
                        }
                     }
                 }
+                loop {
+                    if (executing_queue_count + address_book.provisioning_trackers.len()) >= max_executing_queue_count {
+                        //trace!("skip scheduling; outgoing queue full");
+                        break;
+                    }
+
+                    let prefer_immediate = address_book.provisioning_trackers.len()/4 > executing_queue_count;
+                    let maybe_ee =
+                        Self::schedule_next_execution(&task_sender, runnable_queue, address_book, &mut contended_count, prefer_immediate, &sequence_time, &mut queue_clock, &mut execute_clock);
+
+                    if let Some(ee) = maybe_ee {
+                        //trace!("send to execute");
+                        executing_queue_count += 1;
+
+                        to_execute_substage.send(ee).unwrap();
+                    } else {
+                        break;
+                    }
+                }
             }
         }
     }

From 85b49652acd402cb0cb0bc4d3dc68e04bdff026c Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 26 Aug 2022 15:24:57 +0900
Subject: [PATCH 1427/3199] save

---
 scheduler/src/lib.rs | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 67fdbc99ac22ed..c60fb5cf63d91b 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1086,11 +1086,10 @@ impl ScheduleStage {
                 }
             }
 
-            loop {
-
                 from_len = from.len();
                 from_exec_len = from_exec.len();
 
+            loop {
                 if from_len == 0 && from_exec_len == 0 {
                    trace!("select: back to");
                    break;

From d878a0f17a32c43901ba0794f916b4a879352f8e Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 26 Aug 2022 15:26:21 +0900
Subject: [PATCH 1428/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index c60fb5cf63d91b..8497e3fac37891 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1086,10 +1086,10 @@ impl ScheduleStage {
                 }
             }
 
+            loop {
                 from_len = from.len();
                 from_exec_len = from_exec.len();
 
-            loop {
                 if from_len == 0 && from_exec_len == 0 {
                    trace!("select: back to");
                    break;

From 7b2c4d3693aa25f23c13cf71737c14c6a558028a Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 26 Aug 2022 15:28:47 +0900
Subject: [PATCH 1429/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 8497e3fac37891..c60fb5cf63d91b 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1086,10 +1086,10 @@ impl ScheduleStage {
                 }
             }
 
-            loop {
                 from_len = from.len();
                 from_exec_len = from_exec.len();
 
+            loop {
                 if from_len == 0 && from_exec_len == 0 {
                    trace!("select: back to");
                    break;

From c0b504876bf41b8601dff9bf1776a0cb4cb1b853 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 26 Aug 2022 15:29:37 +0900
Subject: [PATCH 1430/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index c60fb5cf63d91b..8497e3fac37891 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1086,10 +1086,10 @@ impl ScheduleStage {
                 }
             }
 
+            loop {
                 from_len = from.len();
                 from_exec_len = from_exec.len();
 
-            loop {
                 if from_len == 0 && from_exec_len == 0 {
                    trace!("select: back to");
                    break;

From 8da4315561a076102ec8a8fd88f7158d22f98b83 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 26 Aug 2022 15:30:50 +0900
Subject: [PATCH 1431/3199] save

---
 scheduler/src/lib.rs | 24 ++++++++++++------------
 1 file changed, 12 insertions(+), 12 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 8497e3fac37891..c5b7ddc58890f1 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1094,18 +1094,6 @@ impl ScheduleStage {
                    trace!("select: back to");
                    break;
                 } else {
-                    if from_len > 0 {
-                       trace!("select3: {} {}", from_len, from_exec_len);
-                       let task = from.recv().unwrap();
-                       Self::register_runnable_task(task, runnable_queue, &mut current_unique_key, &mut sequence_time);
-                       from_len -= 1;
-                       if from_len == 0 {
-                           from_len = from.len();
-                           if from_len > 0 {
-                               trace!("select3: refill {} {}", from_len, from_exec_len);
-                           }
-                       }
-                    }
                     if from_exec_len > 0 {
                        trace!("select4: {} {}", from_len, from_exec_len);
                         let mut processed_execution_environment = from_exec.recv().unwrap();
@@ -1123,6 +1111,18 @@ impl ScheduleStage {
                            }
                        }
                     }
+                    if from_len > 0 {
+                       trace!("select3: {} {}", from_len, from_exec_len);
+                       let task = from.recv().unwrap();
+                       Self::register_runnable_task(task, runnable_queue, &mut current_unique_key, &mut sequence_time);
+                       from_len -= 1;
+                       if from_len == 0 {
+                           from_len = from.len();
+                           if from_len > 0 {
+                               trace!("select3: refill {} {}", from_len, from_exec_len);
+                           }
+                       }
+                    }
                 }
                 loop {
                     if (executing_queue_count + address_book.provisioning_trackers.len()) >= max_executing_queue_count {

From 0232c0139de56766d301e22129a29136e71a19e8 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 26 Aug 2022 15:31:44 +0900
Subject: [PATCH 1432/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index c5b7ddc58890f1..b2794848be311d 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1086,10 +1086,10 @@ impl ScheduleStage {
                 }
             }
 
-            loop {
                 from_len = from.len();
                 from_exec_len = from_exec.len();
 
+            loop {
                 if from_len == 0 && from_exec_len == 0 {
                    trace!("select: back to");
                    break;

From c794af65cc6cbd6b475a3bab13bd2f5f1866189d Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 26 Aug 2022 15:32:24 +0900
Subject: [PATCH 1433/3199] save

---
 scheduler/src/lib.rs | 16 +---------------
 1 file changed, 1 insertion(+), 15 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index b2794848be311d..bc38a0c3b0942d 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1086,10 +1086,10 @@ impl ScheduleStage {
                 }
             }
 
+            loop {
                 from_len = from.len();
                 from_exec_len = from_exec.len();
 
-            loop {
                 if from_len == 0 && from_exec_len == 0 {
                    trace!("select: back to");
                    break;
@@ -1103,25 +1103,11 @@ impl ScheduleStage {
                         Self::commit_completed_execution(&mut processed_execution_environment, address_book, &mut execute_clock);
                         // async-ly propagate the result to rpc subsystems
                         to_next_stage.send(processed_execution_environment).unwrap();
-                       from_exec_len -= 1;
-                       if from_exec_len == 0 {
-                           from_exec_len = from_exec.len();
-                           if from_exec_len > 0 {
-                               trace!("select3: refill {} {}", from_len, from_exec_len);
-                           }
-                       }
                     }
                     if from_len > 0 {
                        trace!("select3: {} {}", from_len, from_exec_len);
                        let task = from.recv().unwrap();
                        Self::register_runnable_task(task, runnable_queue, &mut current_unique_key, &mut sequence_time);
-                       from_len -= 1;
-                       if from_len == 0 {
-                           from_len = from.len();
-                           if from_len > 0 {
-                               trace!("select3: refill {} {}", from_len, from_exec_len);
-                           }
-                       }
                     }
                 }
                 loop {

From 6463382fea4df5143012acb0215a15c172097d38 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 26 Aug 2022 15:32:50 +0900
Subject: [PATCH 1434/3199] save

---
 scheduler/src/lib.rs | 14 ++++++++++++++
 1 file changed, 14 insertions(+)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index bc38a0c3b0942d..c5b7ddc58890f1 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1103,11 +1103,25 @@ impl ScheduleStage {
                         Self::commit_completed_execution(&mut processed_execution_environment, address_book, &mut execute_clock);
                         // async-ly propagate the result to rpc subsystems
                         to_next_stage.send(processed_execution_environment).unwrap();
+                       from_exec_len -= 1;
+                       if from_exec_len == 0 {
+                           from_exec_len = from_exec.len();
+                           if from_exec_len > 0 {
+                               trace!("select3: refill {} {}", from_len, from_exec_len);
+                           }
+                       }
                     }
                     if from_len > 0 {
                        trace!("select3: {} {}", from_len, from_exec_len);
                        let task = from.recv().unwrap();
                        Self::register_runnable_task(task, runnable_queue, &mut current_unique_key, &mut sequence_time);
+                       from_len -= 1;
+                       if from_len == 0 {
+                           from_len = from.len();
+                           if from_len > 0 {
+                               trace!("select3: refill {} {}", from_len, from_exec_len);
+                           }
+                       }
                     }
                 }
                 loop {

From 2244acd5375830f7594be58d50135069d6c7b8af Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 26 Aug 2022 15:34:06 +0900
Subject: [PATCH 1435/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index c5b7ddc58890f1..b2794848be311d 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1086,10 +1086,10 @@ impl ScheduleStage {
                 }
             }
 
-            loop {
                 from_len = from.len();
                 from_exec_len = from_exec.len();
 
+            loop {
                 if from_len == 0 && from_exec_len == 0 {
                    trace!("select: back to");
                    break;

From 21e8b0b364acc025ba04b09bdf431b939d871f8c Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 26 Aug 2022 15:35:19 +0900
Subject: [PATCH 1436/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index b2794848be311d..c5b7ddc58890f1 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1086,10 +1086,10 @@ impl ScheduleStage {
                 }
             }
 
+            loop {
                 from_len = from.len();
                 from_exec_len = from_exec.len();
 
-            loop {
                 if from_len == 0 && from_exec_len == 0 {
                    trace!("select: back to");
                    break;

From d11a42721c92a81802146ae8607fad1cecc3d5b2 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 26 Aug 2022 15:35:47 +0900
Subject: [PATCH 1437/3199] save

---
 scheduler/src/lib.rs | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index c5b7ddc58890f1..97b546adb4c635 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1049,11 +1049,6 @@ impl ScheduleStage {
             //trace!("schedule_once (from: {}, to: {}, runnnable: {}, contended: {}, (immediate+provisional)/max: ({}+{})/{}) active from contended: {}!", from_len, to_execute_substage.len(), runnable_queue.task_count(), contended_count, executing_queue_count, address_book.provisioning_trackers.len(), max_executing_queue_count, address_book.uncontended_task_ids.len());
 
             crossbeam_channel::select! {
-               recv(from) -> maybe_from => {
-                   //trace!("select1: {} {}", from.len(), from_exec.len());
-                   let task = maybe_from.unwrap();
-                   Self::register_runnable_task(task, runnable_queue, &mut current_unique_key, &mut sequence_time);
-               }
                recv(from_exec) -> maybe_from_exec => {
                    //trace!("select2: {} {}", from.len(), from_exec.len());
                    let mut processed_execution_environment = maybe_from_exec.unwrap();
@@ -1064,6 +1059,11 @@ impl ScheduleStage {
                     // async-ly propagate the result to rpc subsystems
                     to_next_stage.send(processed_execution_environment).unwrap();
                }
+               recv(from) -> maybe_from => {
+                   //trace!("select1: {} {}", from.len(), from_exec.len());
+                   let task = maybe_from.unwrap();
+                   Self::register_runnable_task(task, runnable_queue, &mut current_unique_key, &mut sequence_time);
+               }
             }
 
             loop {

From 096d19348de3c5a7af51e7d3b1bcc8d19735c577 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 26 Aug 2022 15:37:33 +0900
Subject: [PATCH 1438/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 97b546adb4c635..54a0f12030717d 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1086,10 +1086,10 @@ impl ScheduleStage {
                 }
             }
 
-            loop {
                 from_len = from.len();
                 from_exec_len = from_exec.len();
 
+            loop {
                 if from_len == 0 && from_exec_len == 0 {
                    trace!("select: back to");
                    break;

From 60d5ad367e798d29a03e3b3f6585582321e5ebf7 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 26 Aug 2022 15:38:17 +0900
Subject: [PATCH 1439/3199] save

---
 scheduler/src/lib.rs | 24 ++++++++++++------------
 1 file changed, 12 insertions(+), 12 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 54a0f12030717d..74ea632da8e6b5 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1094,6 +1094,18 @@ impl ScheduleStage {
                    trace!("select: back to");
                    break;
                 } else {
+                    if from_len > 0 {
+                       trace!("select3: {} {}", from_len, from_exec_len);
+                       let task = from.recv().unwrap();
+                       Self::register_runnable_task(task, runnable_queue, &mut current_unique_key, &mut sequence_time);
+                       from_len -= 1;
+                       if from_len == 0 {
+                           from_len = from.len();
+                           if from_len > 0 {
+                               trace!("select3: refill {} {}", from_len, from_exec_len);
+                           }
+                       }
+                    }
                     if from_exec_len > 0 {
                        trace!("select4: {} {}", from_len, from_exec_len);
                         let mut processed_execution_environment = from_exec.recv().unwrap();
@@ -1111,18 +1123,6 @@ impl ScheduleStage {
                            }
                        }
                     }
-                    if from_len > 0 {
-                       trace!("select3: {} {}", from_len, from_exec_len);
-                       let task = from.recv().unwrap();
-                       Self::register_runnable_task(task, runnable_queue, &mut current_unique_key, &mut sequence_time);
-                       from_len -= 1;
-                       if from_len == 0 {
-                           from_len = from.len();
-                           if from_len > 0 {
-                               trace!("select3: refill {} {}", from_len, from_exec_len);
-                           }
-                       }
-                    }
                 }
                 loop {
                     if (executing_queue_count + address_book.provisioning_trackers.len()) >= max_executing_queue_count {

From 28981115e65bb046d90ef48318bd7aa4f5477724 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 26 Aug 2022 15:38:53 +0900
Subject: [PATCH 1440/3199] save

---
 scheduler/src/lib.rs | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 74ea632da8e6b5..c60fb5cf63d91b 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1049,6 +1049,11 @@ impl ScheduleStage {
             //trace!("schedule_once (from: {}, to: {}, runnnable: {}, contended: {}, (immediate+provisional)/max: ({}+{})/{}) active from contended: {}!", from_len, to_execute_substage.len(), runnable_queue.task_count(), contended_count, executing_queue_count, address_book.provisioning_trackers.len(), max_executing_queue_count, address_book.uncontended_task_ids.len());
 
             crossbeam_channel::select! {
+               recv(from) -> maybe_from => {
+                   //trace!("select1: {} {}", from.len(), from_exec.len());
+                   let task = maybe_from.unwrap();
+                   Self::register_runnable_task(task, runnable_queue, &mut current_unique_key, &mut sequence_time);
+               }
                recv(from_exec) -> maybe_from_exec => {
                    //trace!("select2: {} {}", from.len(), from_exec.len());
                    let mut processed_execution_environment = maybe_from_exec.unwrap();
@@ -1059,11 +1064,6 @@ impl ScheduleStage {
                     // async-ly propagate the result to rpc subsystems
                     to_next_stage.send(processed_execution_environment).unwrap();
                }
-               recv(from) -> maybe_from => {
-                   //trace!("select1: {} {}", from.len(), from_exec.len());
-                   let task = maybe_from.unwrap();
-                   Self::register_runnable_task(task, runnable_queue, &mut current_unique_key, &mut sequence_time);
-               }
             }
 
             loop {

From 45dfcd35fe6e3b33a6223bbf87b88f7d1cfef642 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 26 Aug 2022 15:39:47 +0900
Subject: [PATCH 1441/3199] save

---
 scheduler/src/lib.rs | 49 ++++++++++++++------------------------------
 1 file changed, 15 insertions(+), 34 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index c60fb5cf63d91b..3f9fea0068b5b2 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1067,29 +1067,29 @@ impl ScheduleStage {
             }
 
             loop {
-                if (executing_queue_count + address_book.provisioning_trackers.len()) >= max_executing_queue_count {
-                    //trace!("skip scheduling; outgoing queue full");
-                    break;
-                }
+                loop {
+                    if (executing_queue_count + address_book.provisioning_trackers.len()) >= max_executing_queue_count {
+                        //trace!("skip scheduling; outgoing queue full");
+                        break;
+                    }
 
-                let prefer_immediate = address_book.provisioning_trackers.len()/4 > executing_queue_count;
-                let maybe_ee =
-                    Self::schedule_next_execution(&task_sender, runnable_queue, address_book, &mut contended_count, prefer_immediate, &sequence_time, &mut queue_clock, &mut execute_clock);
+                    let prefer_immediate = address_book.provisioning_trackers.len()/4 > executing_queue_count;
+                    let maybe_ee =
+                        Self::schedule_next_execution(&task_sender, runnable_queue, address_book, &mut contended_count, prefer_immediate, &sequence_time, &mut queue_clock, &mut execute_clock);
 
-                if let Some(ee) = maybe_ee {
-                    //trace!("send to execute");
-                    executing_queue_count += 1;
+                    if let Some(ee) = maybe_ee {
+                        //trace!("send to execute");
+                        executing_queue_count += 1;
 
-                    to_execute_substage.send(ee).unwrap();
-                } else {
-                    break;
+                        to_execute_substage.send(ee).unwrap();
+                    } else {
+                        break;
+                    }
                 }
-            }
 
                 from_len = from.len();
                 from_exec_len = from_exec.len();
 
-            loop {
                 if from_len == 0 && from_exec_len == 0 {
                    trace!("select: back to");
                    break;
@@ -1124,25 +1124,6 @@ impl ScheduleStage {
                        }
                     }
                 }
-                loop {
-                    if (executing_queue_count + address_book.provisioning_trackers.len()) >= max_executing_queue_count {
-                        //trace!("skip scheduling; outgoing queue full");
-                        break;
-                    }
-
-                    let prefer_immediate = address_book.provisioning_trackers.len()/4 > executing_queue_count;
-                    let maybe_ee =
-                        Self::schedule_next_execution(&task_sender, runnable_queue, address_book, &mut contended_count, prefer_immediate, &sequence_time, &mut queue_clock, &mut execute_clock);
-
-                    if let Some(ee) = maybe_ee {
-                        //trace!("send to execute");
-                        executing_queue_count += 1;
-
-                        to_execute_substage.send(ee).unwrap();
-                    } else {
-                        break;
-                    }
-                }
             }
         }
     }

From 15d25088cdf332ba7d3d83d40ac4dd49047a8ea8 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 26 Aug 2022 15:40:39 +0900
Subject: [PATCH 1442/3199] save

---
 scheduler/src/lib.rs | 27 +++++++++++++++++++++++----
 1 file changed, 23 insertions(+), 4 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 3f9fea0068b5b2..9696ab57d27b45 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1066,6 +1066,29 @@ impl ScheduleStage {
                }
             }
 
+            loop {
+                if (executing_queue_count + address_book.provisioning_trackers.len()) >= max_executing_queue_count {
+                    //trace!("skip scheduling; outgoing queue full");
+                    break;
+                }
+
+                let prefer_immediate = address_book.provisioning_trackers.len()/4 > executing_queue_count;
+                let maybe_ee =
+                    Self::schedule_next_execution(&task_sender, runnable_queue, address_book, &mut contended_count, prefer_immediate, &sequence_time, &mut queue_clock, &mut execute_clock);
+
+                if let Some(ee) = maybe_ee {
+                    //trace!("send to execute");
+                    executing_queue_count += 1;
+
+                    to_execute_substage.send(ee).unwrap();
+                } else {
+                    break;
+                }
+            }
+
+                from_len = from.len();
+                from_exec_len = from_exec.len();
+
             loop {
                 loop {
                     if (executing_queue_count + address_book.provisioning_trackers.len()) >= max_executing_queue_count {
@@ -1086,10 +1109,6 @@ impl ScheduleStage {
                         break;
                     }
                 }
-
-                from_len = from.len();
-                from_exec_len = from_exec.len();
-
                 if from_len == 0 && from_exec_len == 0 {
                    trace!("select: back to");
                    break;

From 0d35a838764f6ef53d0451abbda8ffa8492049f5 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 26 Aug 2022 15:42:29 +0900
Subject: [PATCH 1443/3199] save

---
 scheduler/src/lib.rs | 74 ++++++++++++++++++++++----------------------
 1 file changed, 37 insertions(+), 37 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 9696ab57d27b45..97b546adb4c635 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1049,11 +1049,6 @@ impl ScheduleStage {
             //trace!("schedule_once (from: {}, to: {}, runnnable: {}, contended: {}, (immediate+provisional)/max: ({}+{})/{}) active from contended: {}!", from_len, to_execute_substage.len(), runnable_queue.task_count(), contended_count, executing_queue_count, address_book.provisioning_trackers.len(), max_executing_queue_count, address_book.uncontended_task_ids.len());
 
             crossbeam_channel::select! {
-               recv(from) -> maybe_from => {
-                   //trace!("select1: {} {}", from.len(), from_exec.len());
-                   let task = maybe_from.unwrap();
-                   Self::register_runnable_task(task, runnable_queue, &mut current_unique_key, &mut sequence_time);
-               }
                recv(from_exec) -> maybe_from_exec => {
                    //trace!("select2: {} {}", from.len(), from_exec.len());
                    let mut processed_execution_environment = maybe_from_exec.unwrap();
@@ -1064,6 +1059,11 @@ impl ScheduleStage {
                     // async-ly propagate the result to rpc subsystems
                     to_next_stage.send(processed_execution_environment).unwrap();
                }
+               recv(from) -> maybe_from => {
+                   //trace!("select1: {} {}", from.len(), from_exec.len());
+                   let task = maybe_from.unwrap();
+                   Self::register_runnable_task(task, runnable_queue, &mut current_unique_key, &mut sequence_time);
+               }
             }
 
             loop {
@@ -1086,45 +1086,14 @@ impl ScheduleStage {
                 }
             }
 
+            loop {
                 from_len = from.len();
                 from_exec_len = from_exec.len();
 
-            loop {
-                loop {
-                    if (executing_queue_count + address_book.provisioning_trackers.len()) >= max_executing_queue_count {
-                        //trace!("skip scheduling; outgoing queue full");
-                        break;
-                    }
-
-                    let prefer_immediate = address_book.provisioning_trackers.len()/4 > executing_queue_count;
-                    let maybe_ee =
-                        Self::schedule_next_execution(&task_sender, runnable_queue, address_book, &mut contended_count, prefer_immediate, &sequence_time, &mut queue_clock, &mut execute_clock);
-
-                    if let Some(ee) = maybe_ee {
-                        //trace!("send to execute");
-                        executing_queue_count += 1;
-
-                        to_execute_substage.send(ee).unwrap();
-                    } else {
-                        break;
-                    }
-                }
                 if from_len == 0 && from_exec_len == 0 {
                    trace!("select: back to");
                    break;
                 } else {
-                    if from_len > 0 {
-                       trace!("select3: {} {}", from_len, from_exec_len);
-                       let task = from.recv().unwrap();
-                       Self::register_runnable_task(task, runnable_queue, &mut current_unique_key, &mut sequence_time);
-                       from_len -= 1;
-                       if from_len == 0 {
-                           from_len = from.len();
-                           if from_len > 0 {
-                               trace!("select3: refill {} {}", from_len, from_exec_len);
-                           }
-                       }
-                    }
                     if from_exec_len > 0 {
                        trace!("select4: {} {}", from_len, from_exec_len);
                         let mut processed_execution_environment = from_exec.recv().unwrap();
@@ -1142,6 +1111,37 @@ impl ScheduleStage {
                            }
                        }
                     }
+                    if from_len > 0 {
+                       trace!("select3: {} {}", from_len, from_exec_len);
+                       let task = from.recv().unwrap();
+                       Self::register_runnable_task(task, runnable_queue, &mut current_unique_key, &mut sequence_time);
+                       from_len -= 1;
+                       if from_len == 0 {
+                           from_len = from.len();
+                           if from_len > 0 {
+                               trace!("select3: refill {} {}", from_len, from_exec_len);
+                           }
+                       }
+                    }
+                }
+                loop {
+                    if (executing_queue_count + address_book.provisioning_trackers.len()) >= max_executing_queue_count {
+                        //trace!("skip scheduling; outgoing queue full");
+                        break;
+                    }
+
+                    let prefer_immediate = address_book.provisioning_trackers.len()/4 > executing_queue_count;
+                    let maybe_ee =
+                        Self::schedule_next_execution(&task_sender, runnable_queue, address_book, &mut contended_count, prefer_immediate, &sequence_time, &mut queue_clock, &mut execute_clock);
+
+                    if let Some(ee) = maybe_ee {
+                        //trace!("send to execute");
+                        executing_queue_count += 1;
+
+                        to_execute_substage.send(ee).unwrap();
+                    } else {
+                        break;
+                    }
                 }
             }
         }

From bbd75e6d963326d9bbf66e2d8e06b18ea9980bd3 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 26 Aug 2022 15:42:53 +0900
Subject: [PATCH 1444/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 97b546adb4c635..54a0f12030717d 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1086,10 +1086,10 @@ impl ScheduleStage {
                 }
             }
 
-            loop {
                 from_len = from.len();
                 from_exec_len = from_exec.len();
 
+            loop {
                 if from_len == 0 && from_exec_len == 0 {
                    trace!("select: back to");
                    break;

From 7701ce4733d31915e6ec98d70d0498ec60b950db Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 26 Aug 2022 15:43:40 +0900
Subject: [PATCH 1445/3199] save

---
 scheduler/src/lib.rs | 54 ++++++++++++++++----------------------------
 1 file changed, 20 insertions(+), 34 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 54a0f12030717d..87b56b4abd63e6 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1086,10 +1086,29 @@ impl ScheduleStage {
                 }
             }
 
+            loop {
+                loop {
+                    if (executing_queue_count + address_book.provisioning_trackers.len()) >= max_executing_queue_count {
+                        //trace!("skip scheduling; outgoing queue full");
+                        break;
+                    }
+
+                    let prefer_immediate = address_book.provisioning_trackers.len()/4 > executing_queue_count;
+                    let maybe_ee =
+                        Self::schedule_next_execution(&task_sender, runnable_queue, address_book, &mut contended_count, prefer_immediate, &sequence_time, &mut queue_clock, &mut execute_clock);
+
+                    if let Some(ee) = maybe_ee {
+                        //trace!("send to execute");
+                        executing_queue_count += 1;
+
+                        to_execute_substage.send(ee).unwrap();
+                    } else {
+                        break;
+                    }
+                }
                 from_len = from.len();
                 from_exec_len = from_exec.len();
 
-            loop {
                 if from_len == 0 && from_exec_len == 0 {
                    trace!("select: back to");
                    break;
@@ -1103,44 +1122,11 @@ impl ScheduleStage {
                         Self::commit_completed_execution(&mut processed_execution_environment, address_book, &mut execute_clock);
                         // async-ly propagate the result to rpc subsystems
                         to_next_stage.send(processed_execution_environment).unwrap();
-                       from_exec_len -= 1;
-                       if from_exec_len == 0 {
-                           from_exec_len = from_exec.len();
-                           if from_exec_len > 0 {
-                               trace!("select3: refill {} {}", from_len, from_exec_len);
-                           }
-                       }
                     }
                     if from_len > 0 {
                        trace!("select3: {} {}", from_len, from_exec_len);
                        let task = from.recv().unwrap();
                        Self::register_runnable_task(task, runnable_queue, &mut current_unique_key, &mut sequence_time);
-                       from_len -= 1;
-                       if from_len == 0 {
-                           from_len = from.len();
-                           if from_len > 0 {
-                               trace!("select3: refill {} {}", from_len, from_exec_len);
-                           }
-                       }
-                    }
-                }
-                loop {
-                    if (executing_queue_count + address_book.provisioning_trackers.len()) >= max_executing_queue_count {
-                        //trace!("skip scheduling; outgoing queue full");
-                        break;
-                    }
-
-                    let prefer_immediate = address_book.provisioning_trackers.len()/4 > executing_queue_count;
-                    let maybe_ee =
-                        Self::schedule_next_execution(&task_sender, runnable_queue, address_book, &mut contended_count, prefer_immediate, &sequence_time, &mut queue_clock, &mut execute_clock);
-
-                    if let Some(ee) = maybe_ee {
-                        //trace!("send to execute");
-                        executing_queue_count += 1;
-
-                        to_execute_substage.send(ee).unwrap();
-                    } else {
-                        break;
                     }
                 }
             }

From 1f2c11bd8d4486dee9ffee134632bcfea546f449 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 26 Aug 2022 15:48:19 +0900
Subject: [PATCH 1446/3199] save

---
 scheduler/src/lib.rs | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 87b56b4abd63e6..2ceb5cc681c66f 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1106,6 +1106,7 @@ impl ScheduleStage {
                         break;
                     }
                 }
+                if false {
                 from_len = from.len();
                 from_exec_len = from_exec.len();
 
@@ -1129,6 +1130,7 @@ impl ScheduleStage {
                        Self::register_runnable_task(task, runnable_queue, &mut current_unique_key, &mut sequence_time);
                     }
                 }
+                }
             }
         }
     }

From d934d67fc807a16979701f527b736fbd89f3e048 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 26 Aug 2022 15:48:56 +0900
Subject: [PATCH 1447/3199] save

---
 scheduler/src/lib.rs | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 2ceb5cc681c66f..1458c9105c72db 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1130,6 +1130,8 @@ impl ScheduleStage {
                        Self::register_runnable_task(task, runnable_queue, &mut current_unique_key, &mut sequence_time);
                     }
                 }
+                } else {
+                    break
                 }
             }
         }

From 187e3d4fb0faafe3c17002e57972e665bf017d66 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 26 Aug 2022 15:50:02 +0900
Subject: [PATCH 1448/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 1458c9105c72db..8e08d3393bd4f4 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1106,7 +1106,7 @@ impl ScheduleStage {
                         break;
                     }
                 }
-                if false {
+                if true {
                 from_len = from.len();
                 from_exec_len = from_exec.len();
 

From 2cd36d6ee2d3c1e1c9c4c2a749e12cc0c736f2b9 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 26 Aug 2022 15:52:02 +0900
Subject: [PATCH 1449/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 8e08d3393bd4f4..1458c9105c72db 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1106,7 +1106,7 @@ impl ScheduleStage {
                         break;
                     }
                 }
-                if true {
+                if false {
                 from_len = from.len();
                 from_exec_len = from_exec.len();
 

From 96330fa324b009edcb309889211dfb5ccddaca34 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 26 Aug 2022 15:53:00 +0900
Subject: [PATCH 1450/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 1458c9105c72db..8e08d3393bd4f4 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1106,7 +1106,7 @@ impl ScheduleStage {
                         break;
                     }
                 }
-                if false {
+                if true {
                 from_len = from.len();
                 from_exec_len = from_exec.len();
 

From 9928753df8565f3ab3db7cc46c585a7f44b90ce0 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 26 Aug 2022 15:56:02 +0900
Subject: [PATCH 1451/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 8e08d3393bd4f4..deaa3fcd9df751 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1046,7 +1046,7 @@ impl ScheduleStage {
         let mut from_exec_len = from_exec.len();
 
         loop {
-            //trace!("schedule_once (from: {}, to: {}, runnnable: {}, contended: {}, (immediate+provisional)/max: ({}+{})/{}) active from contended: {}!", from_len, to_execute_substage.len(), runnable_queue.task_count(), contended_count, executing_queue_count, address_book.provisioning_trackers.len(), max_executing_queue_count, address_book.uncontended_task_ids.len());
+            trace!("schedule_once (from: {}, to: {}, runnnable: {}, contended: {}, (immediate+provisional)/max: ({}+{})/{}) active from contended: {}!", from_len, to_execute_substage.len(), runnable_queue.task_count(), contended_count, executing_queue_count, address_book.provisioning_trackers.len(), max_executing_queue_count, address_book.uncontended_task_ids.len());
 
             crossbeam_channel::select! {
                recv(from_exec) -> maybe_from_exec => {

From ad56d179af32666a334354856e75c64f9d3a836d Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 26 Aug 2022 16:37:44 +0900
Subject: [PATCH 1452/3199] save

---
 scheduler/src/lib.rs | 30 +++++++++++++++---------------
 1 file changed, 15 insertions(+), 15 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index deaa3fcd9df751..4e28890a0dd38c 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -167,7 +167,7 @@ pub struct Page {
     current_usage: Usage,
     next_usage: Usage,
     pub contended_unique_weights: TaskIds,
-    provisional_task_ids: std::collections::HashSet<TaskId>,
+    provisional_task_ids: std::collections::HashSet<std::sync::Arc<ProvisioningTracker>>,
     //loaded account from Accounts db
     //comulative_cu for qos; i.e. track serialized cumulative keyed by addresses and bail out block
     //producing as soon as any one of cu from the executing thread reaches to the limit
@@ -201,17 +201,17 @@ type AddressMapEntry<'a> = dashmap::mapref::entry::Entry<'a, Pubkey, PageRc>;
 pub struct AddressBook {
     book: AddressMap,
     uncontended_task_ids: WeightedTaskIds,
-    provisioning_trackers: std::collections::HashMap<UniqueWeight, (ProvisioningTracker, TaskInQueue)>, 
     fulfilled_provisional_task_ids: WeightedTaskIds,
 }
 
 struct ProvisioningTracker {
     remaining_count: usize,
+    task: TaskInQueue,
 }
 
 impl ProvisioningTracker {
-    fn new(remaining_count: usize) -> Self {
-        Self { remaining_count }
+    fn new(remaining_count: usize, task: TaskInQueue) -> Self {
+        Self { remaining_count, task }
     }
 
     fn is_fulfilled(&self) -> bool {
@@ -792,15 +792,14 @@ impl ScheduleStage {
                 assert_eq!(unlockable_count, 0);
                 let lock_count = next_task.tx.1.len();
                 trace!("provisional exec: [{}/{}]", provisional_count, lock_count);
-                Self::finalize_lock_for_provisional_execution(
-                    address_book,
-                    &unique_weight,
-                    &mut next_task.tx.1,
-                );
                 *contended_count = contended_count.checked_sub(1).unwrap();
                 next_task.mark_as_uncontended();
                 drop(next_task);
-                address_book.provisioning_trackers.insert(unique_weight, (ProvisioningTracker::new(provisional_count), arc_next_task));
+                Self::finalize_lock_for_provisional_execution(
+                    address_book,
+                    &arc_next_task,
+                    provisional_count,
+                );
 
                 return None;
                 continue;
@@ -826,13 +825,14 @@ impl ScheduleStage {
     #[inline(never)]
     fn finalize_lock_for_provisional_execution(
         address_book: &mut AddressBook,
-        unique_weight: &UniqueWeight,
-        lock_attempts: &mut Vec<LockAttempt>,
+        task: &TaskInQueue,
+        provisional_count: usize,
     ) {
-        for mut l in lock_attempts {
+        let tracker = Arc::new(ProvisioningTracker::new(provisional_count, TaskInQueue::clone(task)));
+        for mut l in task.tx.1.lock_attempts {
             match l.status {
                 LockStatus::Provisional => {
-                    l.target.page_mut().provisional_task_ids.insert(*unique_weight);
+                    l.target.page_mut().provisional_task_ids.insert(Arc::clone(tracker));
                 }
                 LockStatus::Succeded => {
                     // do nothing
@@ -898,7 +898,7 @@ impl ScheduleStage {
             }
             if page.current_usage == Usage::Unused && page.next_usage != Usage::Unused {
                 page.switch_to_next_usage();
-                for task_id in std::mem::take(&mut page.provisional_task_ids).into_iter() {
+                for task in std::mem::take(&mut page.provisional_task_ids).into_iter() {
                     match address_book.provisioning_trackers.entry(task_id) {
                         std::collections::hash_map::Entry::Occupied(mut tracker_entry) => {
                             let (tracker, task) = tracker_entry.get_mut();

From ea0600acb6f9da0e4e7c9a29c9200692f3e36d9e Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 26 Aug 2022 16:38:02 +0900
Subject: [PATCH 1453/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 4e28890a0dd38c..c638777badbf6b 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -828,7 +828,7 @@ impl ScheduleStage {
         task: &TaskInQueue,
         provisional_count: usize,
     ) {
-        let tracker = Arc::new(ProvisioningTracker::new(provisional_count, TaskInQueue::clone(task)));
+        let tracker = std::sync::Arc::new(ProvisioningTracker::new(provisional_count, TaskInQueue::clone(task)));
         for mut l in task.tx.1.lock_attempts {
             match l.status {
                 LockStatus::Provisional => {

From d4bb4009835e0be334518165a67754d1004f635b Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 26 Aug 2022 16:38:25 +0900
Subject: [PATCH 1454/3199] save

---
 scheduler/src/lib.rs | 1 +
 1 file changed, 1 insertion(+)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index c638777badbf6b..688930337ccbbe 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -204,6 +204,7 @@ pub struct AddressBook {
     fulfilled_provisional_task_ids: WeightedTaskIds,
 }
 
+#[derive(Debug)]
 struct ProvisioningTracker {
     remaining_count: usize,
     task: TaskInQueue,

From 0f011773d5357d2db58ccdc9e64026a4b67ed99c Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 26 Aug 2022 16:38:40 +0900
Subject: [PATCH 1455/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 688930337ccbbe..24f54b2857692c 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -830,7 +830,7 @@ impl ScheduleStage {
         provisional_count: usize,
     ) {
         let tracker = std::sync::Arc::new(ProvisioningTracker::new(provisional_count, TaskInQueue::clone(task)));
-        for mut l in task.tx.1.lock_attempts {
+        for mut l in task.tx.1 {
             match l.status {
                 LockStatus::Provisional => {
                     l.target.page_mut().provisional_task_ids.insert(Arc::clone(tracker));

From 3a0fee7a7cf319541d101e8b986223c28f2cb8ab Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 26 Aug 2022 16:39:02 +0900
Subject: [PATCH 1456/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 24f54b2857692c..48ffaaf588ef08 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -833,7 +833,7 @@ impl ScheduleStage {
         for mut l in task.tx.1 {
             match l.status {
                 LockStatus::Provisional => {
-                    l.target.page_mut().provisional_task_ids.insert(Arc::clone(tracker));
+                    l.target.page_mut().provisional_task_ids.insert(std::sync::Arc::clone(tracker));
                 }
                 LockStatus::Succeded => {
                     // do nothing

From 5c266240a61a2c395681cbbb75190fab45807777 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 26 Aug 2022 16:39:14 +0900
Subject: [PATCH 1457/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 48ffaaf588ef08..e300296d52cbde 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -833,7 +833,7 @@ impl ScheduleStage {
         for mut l in task.tx.1 {
             match l.status {
                 LockStatus::Provisional => {
-                    l.target.page_mut().provisional_task_ids.insert(std::sync::Arc::clone(tracker));
+                    l.target.page_mut().provisional_task_ids.insert(std::sync::Arc::clone(&tracker));
                 }
                 LockStatus::Succeded => {
                     // do nothing

From 58fb69a96978d28e68dbc4273f837c8b942db48f Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 26 Aug 2022 16:40:12 +0900
Subject: [PATCH 1458/3199] save

---
 scheduler/src/lib.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index e300296d52cbde..d19dd1bd7d17e4 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -167,7 +167,7 @@ pub struct Page {
     current_usage: Usage,
     next_usage: Usage,
     pub contended_unique_weights: TaskIds,
-    provisional_task_ids: std::collections::HashSet<std::sync::Arc<ProvisioningTracker>>,
+    provisional_task_ids: std::collections::HashSet<UniqueWeight, std::sync::Arc<ProvisioningTracker>>,
     //loaded account from Accounts db
     //comulative_cu for qos; i.e. track serialized cumulative keyed by addresses and bail out block
     //producing as soon as any one of cu from the executing thread reaches to the limit
@@ -833,7 +833,7 @@ impl ScheduleStage {
         for mut l in task.tx.1 {
             match l.status {
                 LockStatus::Provisional => {
-                    l.target.page_mut().provisional_task_ids.insert(std::sync::Arc::clone(&tracker));
+                    l.target.page_mut().provisional_task_ids.insert(task.unique_weight, std::sync::Arc::clone(&tracker));
                 }
                 LockStatus::Succeded => {
                     // do nothing

From 346703630e13681605ba114ccf60046a548dfa72 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 26 Aug 2022 16:40:43 +0900
Subject: [PATCH 1459/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index d19dd1bd7d17e4..6b16b24a41bafd 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -167,7 +167,7 @@ pub struct Page {
     current_usage: Usage,
     next_usage: Usage,
     pub contended_unique_weights: TaskIds,
-    provisional_task_ids: std::collections::HashSet<UniqueWeight, std::sync::Arc<ProvisioningTracker>>,
+    provisional_task_ids: std::collections::HashMap<UniqueWeight, std::sync::Arc<ProvisioningTracker>>,
     //loaded account from Accounts db
     //comulative_cu for qos; i.e. track serialized cumulative keyed by addresses and bail out block
     //producing as soon as any one of cu from the executing thread reaches to the limit

From bf8ea4690f969d1807441f26b9121790d58a09ac Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 26 Aug 2022 16:41:44 +0900
Subject: [PATCH 1460/3199] save

---
 scheduler/src/lib.rs | 13 ++-----------
 1 file changed, 2 insertions(+), 11 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 6b16b24a41bafd..c6d9eeba19cf3f 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -899,23 +899,14 @@ impl ScheduleStage {
             }
             if page.current_usage == Usage::Unused && page.next_usage != Usage::Unused {
                 page.switch_to_next_usage();
-                for task in std::mem::take(&mut page.provisional_task_ids).into_iter() {
-                    match address_book.provisioning_trackers.entry(task_id) {
-                        std::collections::hash_map::Entry::Occupied(mut tracker_entry) => {
-                            let (tracker, task) = tracker_entry.get_mut();
+                for tracker in std::mem::take(&mut page.provisional_task_ids).into_iter() {
                             tracker.progress();
                             if tracker.is_fulfilled() {
                                 trace!("provisioning tracker progress: {} => {} (!)", tracker.prev_count(), tracker.count());
-                                let (tracker, task) = tracker_entry.remove();
-                                address_book.fulfilled_provisional_task_ids.insert(task_id, task);
+                                address_book.fulfilled_provisional_task_ids.insert(task.unique_weight, task);
                             } else {
                                 trace!("provisioning tracker progress: {} => {}", tracker.prev_count(), tracker.count());
                             }
-                        },
-                        std::collections::hash_map::Entry::Vacant(_) => {
-                            unreachable!();
-                        }
-                    }
                 }
             }
 

From 5326c24273319e818ae83e3655fecc2de56cc1a3 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 26 Aug 2022 16:42:07 +0900
Subject: [PATCH 1461/3199] save

---
 scheduler/src/lib.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index c6d9eeba19cf3f..9ce9c4c6aac6a0 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -899,11 +899,11 @@ impl ScheduleStage {
             }
             if page.current_usage == Usage::Unused && page.next_usage != Usage::Unused {
                 page.switch_to_next_usage();
-                for tracker in std::mem::take(&mut page.provisional_task_ids).into_iter() {
+                for (unique_weight, tracker) in std::mem::take(&mut page.provisional_task_ids).into_iter() {
                             tracker.progress();
                             if tracker.is_fulfilled() {
                                 trace!("provisioning tracker progress: {} => {} (!)", tracker.prev_count(), tracker.count());
-                                address_book.fulfilled_provisional_task_ids.insert(task.unique_weight, task);
+                                address_book.fulfilled_provisional_task_ids.insert(unique_weight, task);
                             } else {
                                 trace!("provisioning tracker progress: {} => {}", tracker.prev_count(), tracker.count());
                             }

From a8fe3fe46db745073ee8a1f1489ffad9876f4c43 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 26 Aug 2022 16:42:23 +0900
Subject: [PATCH 1462/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 9ce9c4c6aac6a0..42fe2eab612e5b 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -903,7 +903,7 @@ impl ScheduleStage {
                             tracker.progress();
                             if tracker.is_fulfilled() {
                                 trace!("provisioning tracker progress: {} => {} (!)", tracker.prev_count(), tracker.count());
-                                address_book.fulfilled_provisional_task_ids.insert(unique_weight, task);
+                                address_book.fulfilled_provisional_task_ids.insert(unique_weight, tracker.task);
                             } else {
                                 trace!("provisioning tracker progress: {} => {}", tracker.prev_count(), tracker.count());
                             }

From 10b7365c11501494c9a943727a20738330af86de Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 26 Aug 2022 16:43:56 +0900
Subject: [PATCH 1463/3199] save

---
 scheduler/src/lib.rs | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 42fe2eab612e5b..1b2c5526cf4b1b 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -843,7 +843,7 @@ impl ScheduleStage {
                 }
             }
         }
-        trace!("provisioning_trackers: {}", address_book.provisioning_trackers.len());
+        //trace!("provisioning_trackers: {}", address_book.provisioning_trackers.len());
     }
 
     #[inline(never)]
@@ -1059,12 +1059,12 @@ impl ScheduleStage {
             }
 
             loop {
-                if (executing_queue_count + address_book.provisioning_trackers.len()) >= max_executing_queue_count {
+                if (executing_queue_count /*+ address_book.provisioning_trackers.len()*/) >= max_executing_queue_count {
                     //trace!("skip scheduling; outgoing queue full");
                     break;
                 }
 
-                let prefer_immediate = address_book.provisioning_trackers.len()/4 > executing_queue_count;
+                let prefer_immediate = true; //address_book.provisioning_trackers.len()/4 > executing_queue_count;
                 let maybe_ee =
                     Self::schedule_next_execution(&task_sender, runnable_queue, address_book, &mut contended_count, prefer_immediate, &sequence_time, &mut queue_clock, &mut execute_clock);
 
@@ -1080,12 +1080,12 @@ impl ScheduleStage {
 
             loop {
                 loop {
-                    if (executing_queue_count + address_book.provisioning_trackers.len()) >= max_executing_queue_count {
+                    if (executing_queue_count /*+ address_book.provisioning_trackers.len()*/) >= max_executing_queue_count {
                         //trace!("skip scheduling; outgoing queue full");
                         break;
                     }
 
-                    let prefer_immediate = address_book.provisioning_trackers.len()/4 > executing_queue_count;
+                    let prefer_immediate = true; //address_book.provisioning_trackers.len()/4 > executing_queue_count;
                     let maybe_ee =
                         Self::schedule_next_execution(&task_sender, runnable_queue, address_book, &mut contended_count, prefer_immediate, &sequence_time, &mut queue_clock, &mut execute_clock);
 

From afa807b9c119d0cc063ed563e12d7d3bbae8481f Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 26 Aug 2022 16:44:10 +0900
Subject: [PATCH 1464/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 1b2c5526cf4b1b..f7019d9300744b 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1038,7 +1038,7 @@ impl ScheduleStage {
         let mut from_exec_len = from_exec.len();
 
         loop {
-            trace!("schedule_once (from: {}, to: {}, runnnable: {}, contended: {}, (immediate+provisional)/max: ({}+{})/{}) active from contended: {}!", from_len, to_execute_substage.len(), runnable_queue.task_count(), contended_count, executing_queue_count, address_book.provisioning_trackers.len(), max_executing_queue_count, address_book.uncontended_task_ids.len());
+            trace!("schedule_once (from: {}, to: {}, runnnable: {}, contended: {}, (immediate+provisional)/max: ({}+{})/{}) active from contended: {}!", from_len, to_execute_substage.len(), runnable_queue.task_count(), contended_count, executing_queue_count, 0/*address_book.provisioning_trackers.len()*/, max_executing_queue_count, address_book.uncontended_task_ids.len());
 
             crossbeam_channel::select! {
                recv(from_exec) -> maybe_from_exec => {

From 876df35436fb96ceb8a2a7920bc424894d4492e4 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 26 Aug 2022 16:44:27 +0900
Subject: [PATCH 1465/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index f7019d9300744b..491c6b8c8a36fa 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -830,7 +830,7 @@ impl ScheduleStage {
         provisional_count: usize,
     ) {
         let tracker = std::sync::Arc::new(ProvisioningTracker::new(provisional_count, TaskInQueue::clone(task)));
-        for mut l in task.tx.1 {
+        for mut l in task.tx.1.iter() {
             match l.status {
                 LockStatus::Provisional => {
                     l.target.page_mut().provisional_task_ids.insert(task.unique_weight, std::sync::Arc::clone(&tracker));

From e84d30cea40b847f00678d2b05bac469dc1b40ab Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 26 Aug 2022 16:44:41 +0900
Subject: [PATCH 1466/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 491c6b8c8a36fa..c3e4cb5b547669 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -830,7 +830,7 @@ impl ScheduleStage {
         provisional_count: usize,
     ) {
         let tracker = std::sync::Arc::new(ProvisioningTracker::new(provisional_count, TaskInQueue::clone(task)));
-        for mut l in task.tx.1.iter() {
+        for l in task.tx.1.iter() {
             match l.status {
                 LockStatus::Provisional => {
                     l.target.page_mut().provisional_task_ids.insert(task.unique_weight, std::sync::Arc::clone(&tracker));

From 41e726b035903e50a7d03011304164a7e225775a Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 26 Aug 2022 16:45:58 +0900
Subject: [PATCH 1467/3199] save

---
 scheduler/src/lib.rs | 15 ++++++++-------
 1 file changed, 8 insertions(+), 7 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index c3e4cb5b547669..982b3cbc04bdab 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -900,13 +900,14 @@ impl ScheduleStage {
             if page.current_usage == Usage::Unused && page.next_usage != Usage::Unused {
                 page.switch_to_next_usage();
                 for (unique_weight, tracker) in std::mem::take(&mut page.provisional_task_ids).into_iter() {
-                            tracker.progress();
-                            if tracker.is_fulfilled() {
-                                trace!("provisioning tracker progress: {} => {} (!)", tracker.prev_count(), tracker.count());
-                                address_book.fulfilled_provisional_task_ids.insert(unique_weight, tracker.task);
-                            } else {
-                                trace!("provisioning tracker progress: {} => {}", tracker.prev_count(), tracker.count());
-                            }
+                    let tracker = unsafe { TaskInQueue::get_mut_unchecked(&mut tracker) };
+                    tracker.progress();
+                    if tracker.is_fulfilled() {
+                        trace!("provisioning tracker progress: {} => {} (!)", tracker.prev_count(), tracker.count());
+                        address_book.fulfilled_provisional_task_ids.insert(unique_weight, tracker.task);
+                    } else {
+                        trace!("provisioning tracker progress: {} => {}", tracker.prev_count(), tracker.count());
+                    }
                 }
             }
 

From e97ca997f901cff20649d2117099df468b4bda5c Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 26 Aug 2022 16:47:24 +0900
Subject: [PATCH 1468/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 982b3cbc04bdab..dd6cb489612123 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -900,7 +900,7 @@ impl ScheduleStage {
             if page.current_usage == Usage::Unused && page.next_usage != Usage::Unused {
                 page.switch_to_next_usage();
                 for (unique_weight, tracker) in std::mem::take(&mut page.provisional_task_ids).into_iter() {
-                    let tracker = unsafe { TaskInQueue::get_mut_unchecked(&mut tracker) };
+                    let tracker = unsafe { std::sync::Arc<ProvisioningTracker>::get_mut_unchecked(&mut tracker) };
                     tracker.progress();
                     if tracker.is_fulfilled() {
                         trace!("provisioning tracker progress: {} => {} (!)", tracker.prev_count(), tracker.count());

From c9b629cfcabb4850ac9c6d53c7d079a13a7a5251 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 26 Aug 2022 16:47:30 +0900
Subject: [PATCH 1469/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index dd6cb489612123..8d9f8559e45fd2 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -900,7 +900,7 @@ impl ScheduleStage {
             if page.current_usage == Usage::Unused && page.next_usage != Usage::Unused {
                 page.switch_to_next_usage();
                 for (unique_weight, tracker) in std::mem::take(&mut page.provisional_task_ids).into_iter() {
-                    let tracker = unsafe { std::sync::Arc<ProvisioningTracker>::get_mut_unchecked(&mut tracker) };
+                    let tracker = unsafe { std::sync::Arc::<ProvisioningTracker>::get_mut_unchecked(&mut tracker) };
                     tracker.progress();
                     if tracker.is_fulfilled() {
                         trace!("provisioning tracker progress: {} => {} (!)", tracker.prev_count(), tracker.count());

From 7bf536d8bd8beecadf8cea8ae89c14a5bdb454f1 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 26 Aug 2022 16:47:42 +0900
Subject: [PATCH 1470/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 8d9f8559e45fd2..abf980f66b095e 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -899,7 +899,7 @@ impl ScheduleStage {
             }
             if page.current_usage == Usage::Unused && page.next_usage != Usage::Unused {
                 page.switch_to_next_usage();
-                for (unique_weight, tracker) in std::mem::take(&mut page.provisional_task_ids).into_iter() {
+                for (unique_weight, mut tracker) in std::mem::take(&mut page.provisional_task_ids).into_iter() {
                     let tracker = unsafe { std::sync::Arc::<ProvisioningTracker>::get_mut_unchecked(&mut tracker) };
                     tracker.progress();
                     if tracker.is_fulfilled() {

From f99bcb7f816a31fe6bdd533cc920fdb2898a59c4 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 26 Aug 2022 16:48:09 +0900
Subject: [PATCH 1471/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index abf980f66b095e..39c9139a71ba64 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -904,7 +904,7 @@ impl ScheduleStage {
                     tracker.progress();
                     if tracker.is_fulfilled() {
                         trace!("provisioning tracker progress: {} => {} (!)", tracker.prev_count(), tracker.count());
-                        address_book.fulfilled_provisional_task_ids.insert(unique_weight, tracker.task);
+                        address_book.fulfilled_provisional_task_ids.insert(unique_weight, TaskInQueue::clone(&tracker.task));
                     } else {
                         trace!("provisioning tracker progress: {} => {}", tracker.prev_count(), tracker.count());
                     }

From 6dfa4d142f03473d558afd3fd0057a3ad5ac4371 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 26 Aug 2022 16:49:15 +0900
Subject: [PATCH 1472/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 39c9139a71ba64..1dfaa4dd21630c 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -830,7 +830,7 @@ impl ScheduleStage {
         provisional_count: usize,
     ) {
         let tracker = std::sync::Arc::new(ProvisioningTracker::new(provisional_count, TaskInQueue::clone(task)));
-        for l in task.tx.1.iter() {
+        for l in task.tx.1.iter_mut() {
             match l.status {
                 LockStatus::Provisional => {
                     l.target.page_mut().provisional_task_ids.insert(task.unique_weight, std::sync::Arc::clone(&tracker));

From cd4f562d676bddc5d9d197d6708aaf7e8dd58c70 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 26 Aug 2022 16:50:01 +0900
Subject: [PATCH 1473/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 1dfaa4dd21630c..9d6a9caf9402b5 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -795,9 +795,9 @@ impl ScheduleStage {
                 trace!("provisional exec: [{}/{}]", provisional_count, lock_count);
                 *contended_count = contended_count.checked_sub(1).unwrap();
                 next_task.mark_as_uncontended();
-                drop(next_task);
                 Self::finalize_lock_for_provisional_execution(
                     address_book,
+                    next_task,
                     &arc_next_task,
                     provisional_count,
                 );

From eabf4dc52d0d14261f5ad65d2cd7b5403cb0286e Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 26 Aug 2022 16:50:13 +0900
Subject: [PATCH 1474/3199] save

---
 scheduler/src/lib.rs | 1 +
 1 file changed, 1 insertion(+)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 9d6a9caf9402b5..25f877c517a67d 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -826,6 +826,7 @@ impl ScheduleStage {
     #[inline(never)]
     fn finalize_lock_for_provisional_execution(
         address_book: &mut AddressBook,
+        next_task: &mut Task,
         task: &TaskInQueue,
         provisional_count: usize,
     ) {

From fe1e6a5bd684ff4ef6ccc465a05b547ca962e298 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 26 Aug 2022 16:50:28 +0900
Subject: [PATCH 1475/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 25f877c517a67d..a0728da63ce4bf 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -798,7 +798,7 @@ impl ScheduleStage {
                 Self::finalize_lock_for_provisional_execution(
                     address_book,
                     next_task,
-                    &arc_next_task,
+                    &a2,
                     provisional_count,
                 );
 

From 0e495ac263d6039c21af6110afebd106fce13402 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 26 Aug 2022 16:50:48 +0900
Subject: [PATCH 1476/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index a0728da63ce4bf..9a1f632a52dbc2 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -831,7 +831,7 @@ impl ScheduleStage {
         provisional_count: usize,
     ) {
         let tracker = std::sync::Arc::new(ProvisioningTracker::new(provisional_count, TaskInQueue::clone(task)));
-        for l in task.tx.1.iter_mut() {
+        for l in next_task.tx.1.iter_mut() {
             match l.status {
                 LockStatus::Provisional => {
                     l.target.page_mut().provisional_task_ids.insert(task.unique_weight, std::sync::Arc::clone(&tracker));

From 82a0e08cb708ccb7098853892ad05314f7c9c82e Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 26 Aug 2022 16:52:02 +0900
Subject: [PATCH 1477/3199] save

---
 scheduler/src/lib.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 9a1f632a52dbc2..343a5c414cc491 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1066,7 +1066,7 @@ impl ScheduleStage {
                     break;
                 }
 
-                let prefer_immediate = true; //address_book.provisioning_trackers.len()/4 > executing_queue_count;
+                let prefer_immediate = false; //address_book.provisioning_trackers.len()/4 > executing_queue_count;
                 let maybe_ee =
                     Self::schedule_next_execution(&task_sender, runnable_queue, address_book, &mut contended_count, prefer_immediate, &sequence_time, &mut queue_clock, &mut execute_clock);
 
@@ -1087,7 +1087,7 @@ impl ScheduleStage {
                         break;
                     }
 
-                    let prefer_immediate = true; //address_book.provisioning_trackers.len()/4 > executing_queue_count;
+                    let prefer_immediate = false; //address_book.provisioning_trackers.len()/4 > executing_queue_count;
                     let maybe_ee =
                         Self::schedule_next_execution(&task_sender, runnable_queue, address_book, &mut contended_count, prefer_immediate, &sequence_time, &mut queue_clock, &mut execute_clock);
 

From 17ac1425153a3953ec1a0e8a4653e4ab4f405c24 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 26 Aug 2022 17:17:14 +0900
Subject: [PATCH 1478/3199] save

---
 transaction-scheduler-bench/src/main.rs | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/transaction-scheduler-bench/src/main.rs b/transaction-scheduler-bench/src/main.rs
index c002ade1bcea82..1d3ecee88f228b 100644
--- a/transaction-scheduler-bench/src/main.rs
+++ b/transaction-scheduler-bench/src/main.rs
@@ -512,7 +512,8 @@ fn send_packets(
         for vv in packet_batches {
             for v in vv {
                 let p = solana_scheduler::get_transaction_priority_details(&v.0);
-                let p = (p << 32) | (rng.gen::<u64>() & 0x0000_0000_ffff_ffff);
+                //let p = (p << 32) | (rng.gen::<u64>() & 0x0000_0000_ffff_ffff);
+                let p = (rng.gen::<u64>() & 0x0000_0000_ffff_ffff);
                 let t = solana_scheduler::Task::new_for_queue(p, v);
                 //for lock_attempt in t.tx.1.iter() {
                 //    lock_attempt.contended_unique_weights().insert_task(p, solana_scheduler::TaskInQueue::clone(&t));

From 86f52aabbb0ecfbf3dd356bd3816c1d5b4649c86 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 26 Aug 2022 17:19:37 +0900
Subject: [PATCH 1479/3199] save

---
 transaction-scheduler-bench/src/main.rs | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/transaction-scheduler-bench/src/main.rs b/transaction-scheduler-bench/src/main.rs
index 1d3ecee88f228b..e85ca0c28270b9 100644
--- a/transaction-scheduler-bench/src/main.rs
+++ b/transaction-scheduler-bench/src/main.rs
@@ -487,6 +487,7 @@ fn send_packets(
 
     let blockhash = Hash::default();
     let start = Instant::now();
+    let unique_weight = usize::max_value();
 
     loop {
         if exit.load(Ordering::Relaxed) {
@@ -513,7 +514,8 @@ fn send_packets(
             for v in vv {
                 let p = solana_scheduler::get_transaction_priority_details(&v.0);
                 //let p = (p << 32) | (rng.gen::<u64>() & 0x0000_0000_ffff_ffff);
-                let p = (rng.gen::<u64>() & 0x0000_0000_ffff_ffff);
+                let p = (p << 32) | (unique_weight & 0x0000_0000_ffff_ffff);
+                unique_weight -= 1;
                 let t = solana_scheduler::Task::new_for_queue(p, v);
                 //for lock_attempt in t.tx.1.iter() {
                 //    lock_attempt.contended_unique_weights().insert_task(p, solana_scheduler::TaskInQueue::clone(&t));

From b5fbdd57bc5eb5201c3e77dcd9d4b2ac4bcc3c66 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 26 Aug 2022 17:19:45 +0900
Subject: [PATCH 1480/3199] save

---
 transaction-scheduler-bench/src/main.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/transaction-scheduler-bench/src/main.rs b/transaction-scheduler-bench/src/main.rs
index e85ca0c28270b9..b6ad4a639be233 100644
--- a/transaction-scheduler-bench/src/main.rs
+++ b/transaction-scheduler-bench/src/main.rs
@@ -487,7 +487,7 @@ fn send_packets(
 
     let blockhash = Hash::default();
     let start = Instant::now();
-    let unique_weight = usize::max_value();
+    let unique_weight = u64::max_value();
 
     loop {
         if exit.load(Ordering::Relaxed) {

From 7b9a334383e4ba2ae403d6ffcf34e6bd6c914d41 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 26 Aug 2022 17:19:53 +0900
Subject: [PATCH 1481/3199] save

---
 transaction-scheduler-bench/src/main.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/transaction-scheduler-bench/src/main.rs b/transaction-scheduler-bench/src/main.rs
index b6ad4a639be233..499771252c7ae7 100644
--- a/transaction-scheduler-bench/src/main.rs
+++ b/transaction-scheduler-bench/src/main.rs
@@ -487,7 +487,7 @@ fn send_packets(
 
     let blockhash = Hash::default();
     let start = Instant::now();
-    let unique_weight = u64::max_value();
+    let mut unique_weight = u64::max_value();
 
     loop {
         if exit.load(Ordering::Relaxed) {

From 99bf1d79752e182fb2e3b75f65f8f96015d262a9 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 26 Aug 2022 17:21:08 +0900
Subject: [PATCH 1482/3199] save

---
 transaction-scheduler-bench/src/main.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/transaction-scheduler-bench/src/main.rs b/transaction-scheduler-bench/src/main.rs
index 499771252c7ae7..36f7d81a76c19b 100644
--- a/transaction-scheduler-bench/src/main.rs
+++ b/transaction-scheduler-bench/src/main.rs
@@ -417,6 +417,7 @@ fn spawn_packet_senders(
         .unwrap_or(format!("{}", 4))
         .parse::<usize>()
         .unwrap();
+    let unique_weight = std::sync::atomic::AtomicU64::default();
     (0..producer_count)
         .map(|i| {
             let num_accounts = if i == 0 && high_conflict_sender > 0 {
@@ -487,7 +488,6 @@ fn send_packets(
 
     let blockhash = Hash::default();
     let start = Instant::now();
-    let mut unique_weight = u64::max_value();
 
     loop {
         if exit.load(Ordering::Relaxed) {

From 3d336a4bb5a6376a4b823fc7e1e9af5a403e4d5a Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 26 Aug 2022 17:21:39 +0900
Subject: [PATCH 1483/3199] save

---
 transaction-scheduler-bench/src/main.rs | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/transaction-scheduler-bench/src/main.rs b/transaction-scheduler-bench/src/main.rs
index 36f7d81a76c19b..88b7b6a7ab25bc 100644
--- a/transaction-scheduler-bench/src/main.rs
+++ b/transaction-scheduler-bench/src/main.rs
@@ -417,7 +417,7 @@ fn spawn_packet_senders(
         .unwrap_or(format!("{}", 4))
         .parse::<usize>()
         .unwrap();
-    let unique_weight = std::sync::atomic::AtomicU64::default();
+    let unique_weight = std::sync::Arc<std::sync::atomic::AtomicU64>::default();
     (0..producer_count)
         .map(|i| {
             let num_accounts = if i == 0 && high_conflict_sender > 0 {
@@ -426,6 +426,7 @@ fn spawn_packet_senders(
                 accounts.len()
             };
             spawn_packet_sender(
+                unique_weight
                 i,
                 producer_count,
                 Arc::clone(&preloader),

From 8697c130675e42f16f86c50a51e9d227b1569c23 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 26 Aug 2022 17:21:59 +0900
Subject: [PATCH 1484/3199] save

---
 transaction-scheduler-bench/src/main.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/transaction-scheduler-bench/src/main.rs b/transaction-scheduler-bench/src/main.rs
index 88b7b6a7ab25bc..6ab1066391e2a3 100644
--- a/transaction-scheduler-bench/src/main.rs
+++ b/transaction-scheduler-bench/src/main.rs
@@ -417,7 +417,7 @@ fn spawn_packet_senders(
         .unwrap_or(format!("{}", 4))
         .parse::<usize>()
         .unwrap();
-    let unique_weight = std::sync::Arc<std::sync::atomic::AtomicU64>::default();
+    let unique_weight = std::sync::Arc::<std::sync::atomic::AtomicU64>::default();
     (0..producer_count)
         .map(|i| {
             let num_accounts = if i == 0 && high_conflict_sender > 0 {

From efff6109f016cc8954f6f3455d0314d54867d075 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 26 Aug 2022 17:22:41 +0900
Subject: [PATCH 1485/3199] save

---
 transaction-scheduler-bench/src/main.rs | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/transaction-scheduler-bench/src/main.rs b/transaction-scheduler-bench/src/main.rs
index 6ab1066391e2a3..be077802e3c3c8 100644
--- a/transaction-scheduler-bench/src/main.rs
+++ b/transaction-scheduler-bench/src/main.rs
@@ -426,7 +426,7 @@ fn spawn_packet_senders(
                 accounts.len()
             };
             spawn_packet_sender(
-                unique_weight
+                unique_weight.clone(),
                 i,
                 producer_count,
                 Arc::clone(&preloader),
@@ -443,6 +443,7 @@ fn spawn_packet_senders(
 }
 
 fn spawn_packet_sender(
+    unique_weight: std::sync::Arc<std::sync::atomic::AtomicU64>,
     i: usize,
     producer_count: usize,
     preloader: Arc<solana_scheduler::Preloader>,

From be540147fce6878b472b5f009b76f1f2e2392265 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 26 Aug 2022 17:23:52 +0900
Subject: [PATCH 1486/3199] save

---
 transaction-scheduler-bench/src/main.rs | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/transaction-scheduler-bench/src/main.rs b/transaction-scheduler-bench/src/main.rs
index be077802e3c3c8..72ed0b4f2ce537 100644
--- a/transaction-scheduler-bench/src/main.rs
+++ b/transaction-scheduler-bench/src/main.rs
@@ -457,6 +457,7 @@ fn spawn_packet_sender(
 ) -> JoinHandle<()> {
     std::thread::Builder::new().name(format!("sol-producer{:02}", i)).spawn(move || {
         send_packets(
+            unique_weight,
             producer_count,
             preloader,
             metrics,
@@ -471,6 +472,7 @@ fn spawn_packet_sender(
 }
 
 fn send_packets(
+    unique_weight: std::sync::Arc<std::sync::atomic::AtomicU64>,
     producer_count: usize,
     preloader: Arc<solana_scheduler::Preloader>,
     metrics: Arc<TransactionSchedulerBenchMetrics>,
@@ -516,8 +518,8 @@ fn send_packets(
             for v in vv {
                 let p = solana_scheduler::get_transaction_priority_details(&v.0);
                 //let p = (p << 32) | (rng.gen::<u64>() & 0x0000_0000_ffff_ffff);
-                let p = (p << 32) | (unique_weight & 0x0000_0000_ffff_ffff);
-                unique_weight -= 1;
+                let uw = unique_weight.fetch_sub(1, std::sync::atomic::Ordering::SeqCst);
+                let p = (p << 32) | (uw & 0x0000_0000_ffff_ffff);
                 let t = solana_scheduler::Task::new_for_queue(p, v);
                 //for lock_attempt in t.tx.1.iter() {
                 //    lock_attempt.contended_unique_weights().insert_task(p, solana_scheduler::TaskInQueue::clone(&t));

From bb5c91df81d39393569fcc25a2c82d0512490a32 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 26 Aug 2022 17:38:13 +0900
Subject: [PATCH 1487/3199] save

---
 scheduler/src/lib.rs | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 343a5c414cc491..4c160d498d6e30 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -167,7 +167,7 @@ pub struct Page {
     current_usage: Usage,
     next_usage: Usage,
     pub contended_unique_weights: TaskIds,
-    provisional_task_ids: std::collections::HashMap<UniqueWeight, std::sync::Arc<ProvisioningTracker>>,
+    provisional_task_ids: std::collections::Vec<std::sync::Arc<ProvisioningTracker>>,
     //loaded account from Accounts db
     //comulative_cu for qos; i.e. track serialized cumulative keyed by addresses and bail out block
     //producing as soon as any one of cu from the executing thread reaches to the limit
@@ -834,7 +834,7 @@ impl ScheduleStage {
         for l in next_task.tx.1.iter_mut() {
             match l.status {
                 LockStatus::Provisional => {
-                    l.target.page_mut().provisional_task_ids.insert(task.unique_weight, std::sync::Arc::clone(&tracker));
+                    l.target.page_mut().provisional_task_ids.push(std::sync::Arc::clone(&tracker));
                 }
                 LockStatus::Succeded => {
                     // do nothing
@@ -900,12 +900,12 @@ impl ScheduleStage {
             }
             if page.current_usage == Usage::Unused && page.next_usage != Usage::Unused {
                 page.switch_to_next_usage();
-                for (unique_weight, mut tracker) in std::mem::take(&mut page.provisional_task_ids).into_iter() {
+                for mut tracker in std::mem::take(&mut page.provisional_task_ids).into_iter() {
                     let tracker = unsafe { std::sync::Arc::<ProvisioningTracker>::get_mut_unchecked(&mut tracker) };
                     tracker.progress();
                     if tracker.is_fulfilled() {
                         trace!("provisioning tracker progress: {} => {} (!)", tracker.prev_count(), tracker.count());
-                        address_book.fulfilled_provisional_task_ids.insert(unique_weight, TaskInQueue::clone(&tracker.task));
+                        address_book.fulfilled_provisional_task_ids.insert(tracker.task.unique_weight, TaskInQueue::clone(&tracker.task));
                     } else {
                         trace!("provisioning tracker progress: {} => {}", tracker.prev_count(), tracker.count());
                     }

From 6cc5e32b9e992787ecd56736a3ffaf9fba5a3bde Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 26 Aug 2022 17:38:39 +0900
Subject: [PATCH 1488/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 4c160d498d6e30..dc77be7e1df4fa 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -167,7 +167,7 @@ pub struct Page {
     current_usage: Usage,
     next_usage: Usage,
     pub contended_unique_weights: TaskIds,
-    provisional_task_ids: std::collections::Vec<std::sync::Arc<ProvisioningTracker>>,
+    provisional_task_ids: Vec<std::sync::Arc<ProvisioningTracker>>,
     //loaded account from Accounts db
     //comulative_cu for qos; i.e. track serialized cumulative keyed by addresses and bail out block
     //producing as soon as any one of cu from the executing thread reaches to the limit

From fdd8cf40710d00f2b049508316349ad5484bf450 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 26 Aug 2022 18:04:36 +0900
Subject: [PATCH 1489/3199] save

---
 ledger-tool/src/main.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/ledger-tool/src/main.rs b/ledger-tool/src/main.rs
index 17d8a44a50125f..05410cd75bc7de 100644
--- a/ledger-tool/src/main.rs
+++ b/ledger-tool/src/main.rs
@@ -448,11 +448,11 @@ fn output_slot(
                                 }
 
                                 muxed_sender
-                                    .send(solana_scheduler::Multiplexed::FromPrevious((
+                                    .send((
                                         //Weight { ix: weight },
                                         weight,
                                         t,
-                                    )))
+                                    ))
                                     .unwrap();
                                 depth.fetch_add(1, Ordering::Relaxed);
                                 weight -= 1;

From 904b552a3ecf529ae3a20b054ded883fe8decec2 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 26 Aug 2022 18:05:07 +0900
Subject: [PATCH 1490/3199] save

---
 ledger-tool/src/main.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ledger-tool/src/main.rs b/ledger-tool/src/main.rs
index 05410cd75bc7de..69a957fcdab3c2 100644
--- a/ledger-tool/src/main.rs
+++ b/ledger-tool/src/main.rs
@@ -437,7 +437,7 @@ fn output_slot(
                     .spawn(move || {
                         for i in 0..loop_count {
                             error!("started!: {} {}", i, txes.len());
-                            for tx in txes.iter().map(|t| Box::new((t.0.clone(), t.1.iter().map(|l| l.clone_for_test()).collect::<Vec<_>>()))) {
+                            for tx in txes.iter().map(|t| (t.0.clone(), t.1.iter().map(|l| l.clone_for_test()).collect::<Vec<_>>())) {
                                 while depth.load(Ordering::Relaxed) > 10_000 {
                                     std::thread::sleep(std::time::Duration::from_micros(10));
                                 }

From 2b7e5efb8a0ba3c54d75504fe6e0dfbea49d2dcb Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 26 Aug 2022 18:05:21 +0900
Subject: [PATCH 1491/3199] save

---
 ledger-tool/src/main.rs | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/ledger-tool/src/main.rs b/ledger-tool/src/main.rs
index 69a957fcdab3c2..6fe19a141412f0 100644
--- a/ledger-tool/src/main.rs
+++ b/ledger-tool/src/main.rs
@@ -448,11 +448,9 @@ fn output_slot(
                                 }
 
                                 muxed_sender
-                                    .send((
-                                        //Weight { ix: weight },
-                                        weight,
+                                    .send(
                                         t,
-                                    ))
+                                    )
                                     .unwrap();
                                 depth.fetch_add(1, Ordering::Relaxed);
                                 weight -= 1;

From d8e66f5e397dfece191f38b6850d123a111c217a Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 26 Aug 2022 18:09:36 +0900
Subject: [PATCH 1492/3199] save

---
 ledger-tool/src/main.rs | 30 +++++++++++++++++++-----------
 1 file changed, 19 insertions(+), 11 deletions(-)

diff --git a/ledger-tool/src/main.rs b/ledger-tool/src/main.rs
index 6fe19a141412f0..79e2c6ecbe1b8e 100644
--- a/ledger-tool/src/main.rs
+++ b/ledger-tool/src/main.rs
@@ -336,24 +336,36 @@ fn output_slot(
                         let uq = ee.unique_weight;
                         for mut lock_attempt in ee.lock_attempts.iter_mut() {
                             let contended_unique_weights = lock_attempt.contended_unique_weights();
-                            contended_unique_weights.remove_task(&uq);
                             contended_unique_weights.heaviest_task_cursor().map(|mut task_cursor| {
                                 let mut found = true;
-                                assert_ne!(task_cursor.key(), &uq);
+                                let mut removed = false;
                                 let mut task = task_cursor.value();
+                                //task.trace_timestamps("in_exec(initial list)");
                                 while !task.currently_contended() {
+                                    if task_cursor.key() == &uq {
+                                        assert!(should_remove);
+                                        removed = task_cursor.remove();
+                                        assert!(removed);
+                                    }
+                                    if task.already_finished() {
+                                        task_cursor.remove();
+                                    }
                                     if let Some(new_cursor) = task_cursor.prev() {
                                         assert!(new_cursor.key() < task_cursor.key());
-                                        assert_ne!(new_cursor.key(), &uq);
                                         task_cursor = new_cursor;
                                         task = task_cursor.value();
+                                        //task.trace_timestamps("in_exec(subsequent list)");
                                     } else {
                                         found = false;
                                         break;
                                     }
                                 }
+                                if should_remove && !removed {
+                                    contended_unique_weights.remove_task(&uq);
+                                }
                                 found.then(|| solana_scheduler::TaskInQueue::clone(task))
                             }).flatten().map(|task| {
+                                //task.trace_timestamps(&format!("in_exec(heaviest:{})", transaction_batch.task.queue_time_label()));
                                 lock_attempt.heaviest_uncontended = Some(task);
                                 ()
                             });
@@ -443,15 +455,11 @@ fn output_slot(
                                 }
 
                                 let t = solana_scheduler::Task::new_for_queue(weight, tx);
-                                for lock_attempt in t.tx.1.iter() {
-                                    lock_attempt.contended_unique_weights().insert_task(weight, solana_scheduler::TaskInQueue::clone(&t));
-                                }
+                                //for lock_attempt in t.tx.1.iter() {
+                                //    lock_attempt.contended_unique_weights().insert_task(weight, solana_scheduler::TaskInQueue::clone(&t));
+                                //}
 
-                                muxed_sender
-                                    .send(
-                                        t,
-                                    )
-                                    .unwrap();
+                                muxed_sender.send(t).unwrap();
                                 depth.fetch_add(1, Ordering::Relaxed);
                                 weight -= 1;
                             }

From c103d1529b5bafb1e136b95c6cff25ea1afc3add Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 26 Aug 2022 18:09:54 +0900
Subject: [PATCH 1493/3199] save

---
 ledger-tool/src/main.rs | 1 +
 1 file changed, 1 insertion(+)

diff --git a/ledger-tool/src/main.rs b/ledger-tool/src/main.rs
index 79e2c6ecbe1b8e..dd747180ea9577 100644
--- a/ledger-tool/src/main.rs
+++ b/ledger-tool/src/main.rs
@@ -334,6 +334,7 @@ fn output_slot(
                             .unwrap();
                         */
                         let uq = ee.unique_weight;
+                        let should_remove = transaction_batch.task.contention_count > 0;
                         for mut lock_attempt in ee.lock_attempts.iter_mut() {
                             let contended_unique_weights = lock_attempt.contended_unique_weights();
                             contended_unique_weights.heaviest_task_cursor().map(|mut task_cursor| {

From 202a1a8c17ddb683f12e64dd45a9d62543444a2f Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 26 Aug 2022 18:10:06 +0900
Subject: [PATCH 1494/3199] save

---
 ledger-tool/src/main.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ledger-tool/src/main.rs b/ledger-tool/src/main.rs
index dd747180ea9577..1d0945661d9def 100644
--- a/ledger-tool/src/main.rs
+++ b/ledger-tool/src/main.rs
@@ -334,7 +334,7 @@ fn output_slot(
                             .unwrap();
                         */
                         let uq = ee.unique_weight;
-                        let should_remove = transaction_batch.task.contention_count > 0;
+                        let should_remove = ee.task.contention_count > 0;
                         for mut lock_attempt in ee.lock_attempts.iter_mut() {
                             let contended_unique_weights = lock_attempt.contended_unique_weights();
                             contended_unique_weights.heaviest_task_cursor().map(|mut task_cursor| {

From 5d991d0265dc60b9069ea2e0b17d52c47a4cc16b Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 26 Aug 2022 18:25:23 +0900
Subject: [PATCH 1495/3199] save

---
 scheduler/src/lib.rs | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index dc77be7e1df4fa..9da21d8a7cb8a0 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1029,6 +1029,9 @@ impl ScheduleStage {
             let task_receiver = task_receiver.clone();
             let h = std::thread::Builder::new().name(format!("sol-indexer{:02}", thx)).spawn(move || {
                 while let (uw, task, ll) = task_receiver.recv().unwrap() {
+                    if task.already_finished() {
+                        break;
+                    }
                     for lock_attempt in ll {
                         lock_attempt.contended_unique_weights().insert_task(uw, TaskInQueue::clone(&task));
                     }

From ade7f85eb4021956562986fbf313b4e8996f75ca Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 26 Aug 2022 18:27:59 +0900
Subject: [PATCH 1496/3199] save

---
 scheduler/src/lib.rs | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 9da21d8a7cb8a0..88e3676e81b651 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1029,10 +1029,10 @@ impl ScheduleStage {
             let task_receiver = task_receiver.clone();
             let h = std::thread::Builder::new().name(format!("sol-indexer{:02}", thx)).spawn(move || {
                 while let (uw, task, ll) = task_receiver.recv().unwrap() {
-                    if task.already_finished() {
-                        break;
-                    }
                     for lock_attempt in ll {
+                        if task.already_finished() {
+                            break;
+                        }
                         lock_attempt.contended_unique_weights().insert_task(uw, TaskInQueue::clone(&task));
                     }
                 }

From 49c26a46e2eb85fbf64c9c73238ee0e1f0196d15 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 26 Aug 2022 18:45:46 +0900
Subject: [PATCH 1497/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 88e3676e81b651..1d37fec44b9b8a 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -901,7 +901,7 @@ impl ScheduleStage {
             if page.current_usage == Usage::Unused && page.next_usage != Usage::Unused {
                 page.switch_to_next_usage();
                 for mut tracker in std::mem::take(&mut page.provisional_task_ids).into_iter() {
-                    let tracker = unsafe { std::sync::Arc::<ProvisioningTracker>::get_mut_unchecked(&mut tracker) };
+                    let tracker = unsafe { std::sync::Arc::<(ProvisioningTracker, usize)>::get_mut_unchecked(&mut tracker) };
                     tracker.progress();
                     if tracker.is_fulfilled() {
                         trace!("provisioning tracker progress: {} => {} (!)", tracker.prev_count(), tracker.count());

From b989a1453cba33e93df02eb3a0e652602e1b5cc8 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 26 Aug 2022 18:46:17 +0900
Subject: [PATCH 1498/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 1d37fec44b9b8a..32e078a587d7b7 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -901,7 +901,7 @@ impl ScheduleStage {
             if page.current_usage == Usage::Unused && page.next_usage != Usage::Unused {
                 page.switch_to_next_usage();
                 for mut tracker in std::mem::take(&mut page.provisional_task_ids).into_iter() {
-                    let tracker = unsafe { std::sync::Arc::<(ProvisioningTracker, usize)>::get_mut_unchecked(&mut tracker) };
+                    let tracker = unsafe { TaskInQueue::get_mut_unchecked(&mut tracker) };
                     tracker.progress();
                     if tracker.is_fulfilled() {
                         trace!("provisioning tracker progress: {} => {} (!)", tracker.prev_count(), tracker.count());

From bf8ec1c16f08e607e5218f694ffd084da828e94e Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 26 Aug 2022 18:46:42 +0900
Subject: [PATCH 1499/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 32e078a587d7b7..88e3676e81b651 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -901,7 +901,7 @@ impl ScheduleStage {
             if page.current_usage == Usage::Unused && page.next_usage != Usage::Unused {
                 page.switch_to_next_usage();
                 for mut tracker in std::mem::take(&mut page.provisional_task_ids).into_iter() {
-                    let tracker = unsafe { TaskInQueue::get_mut_unchecked(&mut tracker) };
+                    let tracker = unsafe { std::sync::Arc::<ProvisioningTracker>::get_mut_unchecked(&mut tracker) };
                     tracker.progress();
                     if tracker.is_fulfilled() {
                         trace!("provisioning tracker progress: {} => {} (!)", tracker.prev_count(), tracker.count());

From 2785f388e2e48efe609f0559a7c8f39c87e6659c Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 26 Aug 2022 18:49:50 +0900
Subject: [PATCH 1500/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 88e3676e81b651..6a7bc568417b0c 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -776,7 +776,7 @@ impl ScheduleStage {
                     //for lock_attempt in next_task.tx.1.iter() {
                     //    lock_attempt.contended_unique_weights().insert_task(unique_weight, TaskInQueue::clone(&a2));
                     //}
-                    task_sender.send((unique_weight, TaskInQueue::clone(&a2), std::mem::take(&mut next_task.for_indexer))).unwrap();
+                    task_sender.send(TaskInQueue::clone(&a2), std::mem::take(&mut next_task.for_indexer))).unwrap();
                     // maybe run lightweight prune logic on contended_queue here.
                 } else {
                     trace!("relock failed [{}/{}/{}]; remains in contended: {:?} contention: {}", unlockable_count, provisional_count, lock_count, &unique_weight, next_task.contention_count);

From 9a65cf34a01b9354d6a85a4d05a85b3d55fbcde8 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 26 Aug 2022 18:49:59 +0900
Subject: [PATCH 1501/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 6a7bc568417b0c..c9737122f04792 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -776,7 +776,7 @@ impl ScheduleStage {
                     //for lock_attempt in next_task.tx.1.iter() {
                     //    lock_attempt.contended_unique_weights().insert_task(unique_weight, TaskInQueue::clone(&a2));
                     //}
-                    task_sender.send(TaskInQueue::clone(&a2), std::mem::take(&mut next_task.for_indexer))).unwrap();
+                    task_sender.send((TaskInQueue::clone(&a2), std::mem::take(&mut next_task.for_indexer))).unwrap();
                     // maybe run lightweight prune logic on contended_queue here.
                 } else {
                     trace!("relock failed [{}/{}/{}]; remains in contended: {:?} contention: {}", unlockable_count, provisional_count, lock_count, &unique_weight, next_task.contention_count);

From 49e216ebb30baf81ec3803ed5b8bc87df177c533 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 26 Aug 2022 18:50:33 +0900
Subject: [PATCH 1502/3199] save

---
 scheduler/src/lib.rs | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index c9737122f04792..95fe03b6f3adf0 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -716,7 +716,7 @@ impl ScheduleStage {
 
     #[inline(never)]
     fn pop_from_queue_then_lock(
-        task_sender: &crossbeam_channel::Sender<(u64, TaskInQueue, Vec<LockAttempt>)>,
+        task_sender: &crossbeam_channel::Sender<(TaskInQueue, Vec<LockAttempt>)>,
         runnable_queue: &mut TaskQueue,
         address_book: &mut AddressBook,
         contended_count: &mut usize,
@@ -961,7 +961,7 @@ impl ScheduleStage {
 
     #[inline(never)]
     fn schedule_next_execution(
-        task_sender: &crossbeam_channel::Sender<(u64, TaskInQueue, Vec<LockAttempt>)>,
+        task_sender: &crossbeam_channel::Sender<(TaskInQueue, Vec<LockAttempt>)>,
         runnable_queue: &mut TaskQueue,
         address_book: &mut AddressBook,
         contended_count: &mut usize,
@@ -1020,7 +1020,7 @@ impl ScheduleStage {
 
             (&ee_sender, Some(h))
         };
-        let (task_sender, task_receiver) = crossbeam_channel::unbounded::<(u64, TaskInQueue, Vec<LockAttempt>)>();
+        let (task_sender, task_receiver) = crossbeam_channel::unbounded::<(TaskInQueue, Vec<LockAttempt>)>();
         let indexer_count = std::env::var("INDEXER_COUNT")
             .unwrap_or(format!("{}", 4))
             .parse::<usize>()

From 19d28666a9969d105fde70927d28f471a6780e67 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 26 Aug 2022 18:50:54 +0900
Subject: [PATCH 1503/3199] save

---
 scheduler/src/lib.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 95fe03b6f3adf0..9b7073d7376f40 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1028,12 +1028,12 @@ impl ScheduleStage {
         for thx in 0..indexer_count {
             let task_receiver = task_receiver.clone();
             let h = std::thread::Builder::new().name(format!("sol-indexer{:02}", thx)).spawn(move || {
-                while let (uw, task, ll) = task_receiver.recv().unwrap() {
+                while let (task, ll) = task_receiver.recv().unwrap() {
                     for lock_attempt in ll {
                         if task.already_finished() {
                             break;
                         }
-                        lock_attempt.contended_unique_weights().insert_task(uw, TaskInQueue::clone(&task));
+                        lock_attempt.contended_unique_weights().insert_task(task.unique_weight, TaskInQueue::clone(&task));
                     }
                 }
             }).unwrap();

From 3a13fd9d1705bbdde1bd62661bf095333a1c7d93 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 27 Aug 2022 13:27:29 +0900
Subject: [PATCH 1504/3199] save

---
 scheduler/src/lib.rs | 20 --------------------
 1 file changed, 20 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 9b7073d7376f40..e6c6c3316063d8 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1063,26 +1063,6 @@ impl ScheduleStage {
                }
             }
 
-            loop {
-                if (executing_queue_count /*+ address_book.provisioning_trackers.len()*/) >= max_executing_queue_count {
-                    //trace!("skip scheduling; outgoing queue full");
-                    break;
-                }
-
-                let prefer_immediate = false; //address_book.provisioning_trackers.len()/4 > executing_queue_count;
-                let maybe_ee =
-                    Self::schedule_next_execution(&task_sender, runnable_queue, address_book, &mut contended_count, prefer_immediate, &sequence_time, &mut queue_clock, &mut execute_clock);
-
-                if let Some(ee) = maybe_ee {
-                    //trace!("send to execute");
-                    executing_queue_count += 1;
-
-                    to_execute_substage.send(ee).unwrap();
-                } else {
-                    break;
-                }
-            }
-
             loop {
                 loop {
                     if (executing_queue_count /*+ address_book.provisioning_trackers.len()*/) >= max_executing_queue_count {

From be313afe5188ca884cf871036d81b9c1e61355bc Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 27 Aug 2022 13:30:58 +0900
Subject: [PATCH 1505/3199] save

---
 scheduler/src/lib.rs | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index e6c6c3316063d8..7ac18992e8f2a4 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1063,6 +1063,9 @@ impl ScheduleStage {
                }
             }
 
+                from_len = from.len();
+                from_exec_len = from_exec.len();
+
             loop {
                 loop {
                     if (executing_queue_count /*+ address_book.provisioning_trackers.len()*/) >= max_executing_queue_count {
@@ -1084,9 +1087,6 @@ impl ScheduleStage {
                     }
                 }
                 if true {
-                from_len = from.len();
-                from_exec_len = from_exec.len();
-
                 if from_len == 0 && from_exec_len == 0 {
                    trace!("select: back to");
                    break;

From 073212eda2d547fc67d0be29d420d769ab19ac45 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 27 Aug 2022 13:32:34 +0900
Subject: [PATCH 1506/3199] save

---
 scheduler/src/lib.rs | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 7ac18992e8f2a4..6b0d6a48001f34 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1100,11 +1100,15 @@ impl ScheduleStage {
                         Self::commit_completed_execution(&mut processed_execution_environment, address_book, &mut execute_clock);
                         // async-ly propagate the result to rpc subsystems
                         to_next_stage.send(processed_execution_environment).unwrap();
+                    } else {
+                        from_exec_len = from_exec.len();
                     }
                     if from_len > 0 {
                        trace!("select3: {} {}", from_len, from_exec_len);
                        let task = from.recv().unwrap();
                        Self::register_runnable_task(task, runnable_queue, &mut current_unique_key, &mut sequence_time);
+                    } else {
+                        from_len = from.len();
                     }
                 }
                 } else {

From 812543bf4c0539400e04a4de6a14fc014f64a0b3 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 27 Aug 2022 13:40:33 +0900
Subject: [PATCH 1507/3199] save

---
 scheduler/src/lib.rs | 18 ++----------------
 1 file changed, 2 insertions(+), 16 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 6b0d6a48001f34..158cbce2cdf626 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1067,26 +1067,15 @@ impl ScheduleStage {
                 from_exec_len = from_exec.len();
 
             loop {
-                loop {
-                    if (executing_queue_count /*+ address_book.provisioning_trackers.len()*/) >= max_executing_queue_count {
-                        //trace!("skip scheduling; outgoing queue full");
-                        break;
-                    }
-
+                while (executing_queue_count /*+ address_book.provisioning_trackers.len()*/) >= max_executing_queue_count {
                     let prefer_immediate = false; //address_book.provisioning_trackers.len()/4 > executing_queue_count;
-                    let maybe_ee =
-                        Self::schedule_next_execution(&task_sender, runnable_queue, address_book, &mut contended_count, prefer_immediate, &sequence_time, &mut queue_clock, &mut execute_clock);
-
-                    if let Some(ee) = maybe_ee {
-                        //trace!("send to execute");
+                    if let Some(ee) = Self::schedule_next_execution(&task_sender, runnable_queue, address_book, &mut contended_count, prefer_immediate, &sequence_time, &mut queue_clock, &mut execute_clock) {
                         executing_queue_count += 1;
-
                         to_execute_substage.send(ee).unwrap();
                     } else {
                         break;
                     }
                 }
-                if true {
                 if from_len == 0 && from_exec_len == 0 {
                    trace!("select: back to");
                    break;
@@ -1111,9 +1100,6 @@ impl ScheduleStage {
                         from_len = from.len();
                     }
                 }
-                } else {
-                    break
-                }
             }
         }
     }

From 05f455904cca1e1487e8a430aabdab6e79d42f87 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 27 Aug 2022 13:41:18 +0900
Subject: [PATCH 1508/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 158cbce2cdf626..f8ccc5b264bf0a 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1067,7 +1067,7 @@ impl ScheduleStage {
                 from_exec_len = from_exec.len();
 
             loop {
-                while (executing_queue_count /*+ address_book.provisioning_trackers.len()*/) >= max_executing_queue_count {
+                while (executing_queue_count /*+ address_book.provisioning_trackers.len()*/) < max_executing_queue_count {
                     let prefer_immediate = false; //address_book.provisioning_trackers.len()/4 > executing_queue_count;
                     if let Some(ee) = Self::schedule_next_execution(&task_sender, runnable_queue, address_book, &mut contended_count, prefer_immediate, &sequence_time, &mut queue_clock, &mut execute_clock) {
                         executing_queue_count += 1;

From 69ecdabaf595857df7bd92d374090b6e8dc3c08e Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 27 Aug 2022 13:54:40 +0900
Subject: [PATCH 1509/3199] save

---
 scheduler/src/lib.rs | 19 +++++++++++--------
 1 file changed, 11 insertions(+), 8 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index f8ccc5b264bf0a..069e0765af41cb 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1039,9 +1039,6 @@ impl ScheduleStage {
             }).unwrap();
         }
 
-        let mut from_len = from.len();
-        let mut from_exec_len = from_exec.len();
-
         loop {
             trace!("schedule_once (from: {}, to: {}, runnnable: {}, contended: {}, (immediate+provisional)/max: ({}+{})/{}) active from contended: {}!", from_len, to_execute_substage.len(), runnable_queue.task_count(), contended_count, executing_queue_count, 0/*address_book.provisioning_trackers.len()*/, max_executing_queue_count, address_book.uncontended_task_ids.len());
 
@@ -1063,8 +1060,9 @@ impl ScheduleStage {
                }
             }
 
-                from_len = from.len();
-                from_exec_len = from_exec.len();
+            let mut first_iteration = true;
+            let mut from_len = 0;
+            let mut from_exec_len = 0;
 
             loop {
                 while (executing_queue_count /*+ address_book.provisioning_trackers.len()*/) < max_executing_queue_count {
@@ -1076,11 +1074,16 @@ impl ScheduleStage {
                         break;
                     }
                 }
-                if from_len == 0 && from_exec_len == 0 {
+                if first_iteration {
+                    first_iteration = false;
+                    (from_len,  from_exec_len) = (from.len(), from_exec.len());
+                }
+                let (empty_from, empty_from_exec) = (from_len == 0, from_exec_len == 0);
+                if empty_from && empty_from_exec {
                    trace!("select: back to");
                    break;
                 } else {
-                    if from_exec_len > 0 {
+                    if !empty_from_exec {
                        trace!("select4: {} {}", from_len, from_exec_len);
                         let mut processed_execution_environment = from_exec.recv().unwrap();
                         trace!("recv from execute: {:?}", processed_execution_environment.unique_weight);
@@ -1092,7 +1095,7 @@ impl ScheduleStage {
                     } else {
                         from_exec_len = from_exec.len();
                     }
-                    if from_len > 0 {
+                    if !empty_from {
                        trace!("select3: {} {}", from_len, from_exec_len);
                        let task = from.recv().unwrap();
                        Self::register_runnable_task(task, runnable_queue, &mut current_unique_key, &mut sequence_time);

From 9dd67f1608c346db417cc126e14b0c8426f0fa22 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 27 Aug 2022 13:55:02 +0900
Subject: [PATCH 1510/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 069e0765af41cb..1eae6b799b7b36 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1040,7 +1040,7 @@ impl ScheduleStage {
         }
 
         loop {
-            trace!("schedule_once (from: {}, to: {}, runnnable: {}, contended: {}, (immediate+provisional)/max: ({}+{})/{}) active from contended: {}!", from_len, to_execute_substage.len(), runnable_queue.task_count(), contended_count, executing_queue_count, 0/*address_book.provisioning_trackers.len()*/, max_executing_queue_count, address_book.uncontended_task_ids.len());
+            trace!("schedule_once (from: {}, to: {}, runnnable: {}, contended: {}, (immediate+provisional)/max: ({}+{})/{}) active from contended: {}!", from.len(), to_execute_substage.len(), runnable_queue.task_count(), contended_count, executing_queue_count, 0/*address_book.provisioning_trackers.len()*/, max_executing_queue_count, address_book.uncontended_task_ids.len());
 
             crossbeam_channel::select! {
                recv(from_exec) -> maybe_from_exec => {

From 1353c2a1dec07eefdf01d8a27cf9d1044aa009b1 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 27 Aug 2022 13:56:03 +0900
Subject: [PATCH 1511/3199] save

---
 scheduler/src/lib.rs | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 1eae6b799b7b36..3adabf5b22d37d 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1061,8 +1061,7 @@ impl ScheduleStage {
             }
 
             let mut first_iteration = true;
-            let mut from_len = 0;
-            let mut from_exec_len = 0;
+            let (mut from_len, mut from_exec_len) = (0, 0);
 
             loop {
                 while (executing_queue_count /*+ address_book.provisioning_trackers.len()*/) < max_executing_queue_count {

From 3b7d8c7fff574a61d67194e60c305ddc9a0ba304 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 27 Aug 2022 13:59:49 +0900
Subject: [PATCH 1512/3199] save

---
 scheduler/src/lib.rs | 11 -----------
 1 file changed, 11 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 3adabf5b22d37d..8224289ac05d43 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1044,17 +1044,12 @@ impl ScheduleStage {
 
             crossbeam_channel::select! {
                recv(from_exec) -> maybe_from_exec => {
-                   //trace!("select2: {} {}", from.len(), from_exec.len());
                    let mut processed_execution_environment = maybe_from_exec.unwrap();
-                    //trace!("recv from execute: {:?}", processed_execution_environment.unique_weight);
                     executing_queue_count -= 1;
-
                     Self::commit_completed_execution(&mut processed_execution_environment, address_book, &mut execute_clock);
-                    // async-ly propagate the result to rpc subsystems
                     to_next_stage.send(processed_execution_environment).unwrap();
                }
                recv(from) -> maybe_from => {
-                   //trace!("select1: {} {}", from.len(), from_exec.len());
                    let task = maybe_from.unwrap();
                    Self::register_runnable_task(task, runnable_queue, &mut current_unique_key, &mut sequence_time);
                }
@@ -1079,23 +1074,17 @@ impl ScheduleStage {
                 }
                 let (empty_from, empty_from_exec) = (from_len == 0, from_exec_len == 0);
                 if empty_from && empty_from_exec {
-                   trace!("select: back to");
                    break;
                 } else {
                     if !empty_from_exec {
-                       trace!("select4: {} {}", from_len, from_exec_len);
                         let mut processed_execution_environment = from_exec.recv().unwrap();
-                        trace!("recv from execute: {:?}", processed_execution_environment.unique_weight);
                         executing_queue_count -= 1;
-
                         Self::commit_completed_execution(&mut processed_execution_environment, address_book, &mut execute_clock);
-                        // async-ly propagate the result to rpc subsystems
                         to_next_stage.send(processed_execution_environment).unwrap();
                     } else {
                         from_exec_len = from_exec.len();
                     }
                     if !empty_from {
-                       trace!("select3: {} {}", from_len, from_exec_len);
                        let task = from.recv().unwrap();
                        Self::register_runnable_task(task, runnable_queue, &mut current_unique_key, &mut sequence_time);
                     } else {

From 105ffe20284c05c1990539f0d724fa58a1ee0d22 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 27 Aug 2022 14:19:45 +0900
Subject: [PATCH 1513/3199] save

---
 scheduler/src/lib.rs | 27 ++++++++++++++++-----------
 1 file changed, 16 insertions(+), 11 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 8224289ac05d43..3af1888120cfbe 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -723,6 +723,7 @@ impl ScheduleStage {
         prefer_immediate: bool,
         sequence_clock: &usize,
         queue_clock: &mut usize,
+        provisioning_tracker_count: &mut usize,
     ) -> Option<(UniqueWeight, TaskInQueue, Vec<LockAttempt>)> {
         if let Some(mut a) = address_book.fulfilled_provisional_task_ids.pop_last() {
             trace!("expediate pop from provisional queue [rest: {}]", address_book.fulfilled_provisional_task_ids.len());
@@ -795,6 +796,8 @@ impl ScheduleStage {
                 trace!("provisional exec: [{}/{}]", provisional_count, lock_count);
                 *contended_count = contended_count.checked_sub(1).unwrap();
                 next_task.mark_as_uncontended();
+                let tracker = std::sync::Arc::new(ProvisioningTracker::new(provisional_count, TaskInQueue::clone(task)));
+                provisioning_tracker_count += 1;
                 Self::finalize_lock_for_provisional_execution(
                     address_book,
                     next_task,
@@ -830,7 +833,6 @@ impl ScheduleStage {
         task: &TaskInQueue,
         provisional_count: usize,
     ) {
-        let tracker = std::sync::Arc::new(ProvisioningTracker::new(provisional_count, TaskInQueue::clone(task)));
         for l in next_task.tx.1.iter_mut() {
             match l.status {
                 LockStatus::Provisional => {
@@ -860,7 +862,7 @@ impl ScheduleStage {
     }
 
     #[inline(never)]
-    fn unlock_after_execution(address_book: &mut AddressBook, lock_attempts: &mut Vec<LockAttempt>) {
+    fn unlock_after_execution(address_book: &mut AddressBook, lock_attempts: &mut Vec<LockAttempt>, provisioning_tracker_count: &mut usize) {
         for mut l in lock_attempts.into_iter() {
             let newly_uncontended = address_book.reset_lock(&mut l, true);
 
@@ -906,6 +908,7 @@ impl ScheduleStage {
                     if tracker.is_fulfilled() {
                         trace!("provisioning tracker progress: {} => {} (!)", tracker.prev_count(), tracker.count());
                         address_book.fulfilled_provisional_task_ids.insert(tracker.task.unique_weight, TaskInQueue::clone(&tracker.task));
+                        provisioning_tracker_count -= 1;
                     } else {
                         trace!("provisioning tracker progress: {} => {}", tracker.prev_count(), tracker.count());
                     }
@@ -939,7 +942,7 @@ impl ScheduleStage {
     }
 
     #[inline(never)]
-    fn commit_completed_execution(ee: &mut ExecutionEnvironment, address_book: &mut AddressBook, commit_time: &mut usize) {
+    fn commit_completed_execution(ee: &mut ExecutionEnvironment, address_book: &mut AddressBook, commit_time: &mut usize, provisioning_tracker_count: &mut usize) {
         // do par()-ly?
 
         ee.task.record_commit_time(*commit_time);
@@ -947,7 +950,7 @@ impl ScheduleStage {
         //*commit_time = commit_time.checked_add(1).unwrap();
 
         // which order for data race free?: unlocking / marking
-        Self::unlock_after_execution(address_book, &mut ee.lock_attempts);
+        Self::unlock_after_execution(address_book, &mut ee.lock_attempts, provisioning_tracker_count);
         ee.task.mark_as_finished();
 
         // block-wide qos validation will be done here
@@ -969,9 +972,10 @@ impl ScheduleStage {
         sequence_time: &usize,
         queue_clock: &mut usize,
         execute_clock: &mut usize,
+        provisioning_tracker_count: &mut usize,
     ) -> Option<Box<ExecutionEnvironment>> {
         let maybe_ee =
-            Self::pop_from_queue_then_lock(task_sender, runnable_queue, address_book, contended_count, prefer_immediate, sequence_time, queue_clock)
+            Self::pop_from_queue_then_lock(task_sender, runnable_queue, address_book, contended_count, prefer_immediate, sequence_time, queue_clock, provisioning_tracker_count)
                 .map(|(uw, t,ll)| Self::prepare_scheduled_execution(address_book, uw, t, ll, queue_clock, execute_clock));
         maybe_ee
     }
@@ -1000,6 +1004,7 @@ impl ScheduleStage {
         let mut executing_queue_count = 0;
         let mut current_unique_key = u64::max_value();
         let mut contended_count = 0;
+        let mut provisioning_tracker_count = 0;
         let mut sequence_time = 0;
         let mut queue_clock = 0;
         let mut execute_clock = 0;
@@ -1040,13 +1045,13 @@ impl ScheduleStage {
         }
 
         loop {
-            trace!("schedule_once (from: {}, to: {}, runnnable: {}, contended: {}, (immediate+provisional)/max: ({}+{})/{}) active from contended: {}!", from.len(), to_execute_substage.len(), runnable_queue.task_count(), contended_count, executing_queue_count, 0/*address_book.provisioning_trackers.len()*/, max_executing_queue_count, address_book.uncontended_task_ids.len());
+            trace!("schedule_once (from: {}, to: {}, runnnable: {}, contended: {}, (immediate+provisional)/max: ({}+{})/{}) active from contended: {}!", from.len(), to_execute_substage.len(), runnable_queue.task_count(), contended_count, executing_queue_count, provisioning_tracker_count, max_executing_queue_count, address_book.uncontended_task_ids.len());
 
             crossbeam_channel::select! {
                recv(from_exec) -> maybe_from_exec => {
                    let mut processed_execution_environment = maybe_from_exec.unwrap();
                     executing_queue_count -= 1;
-                    Self::commit_completed_execution(&mut processed_execution_environment, address_book, &mut execute_clock);
+                    Self::commit_completed_execution(&mut processed_execution_environment, address_book, &mut execute_clock, &mut provisioning_tracker_count);
                     to_next_stage.send(processed_execution_environment).unwrap();
                }
                recv(from) -> maybe_from => {
@@ -1059,9 +1064,9 @@ impl ScheduleStage {
             let (mut from_len, mut from_exec_len) = (0, 0);
 
             loop {
-                while (executing_queue_count /*+ address_book.provisioning_trackers.len()*/) < max_executing_queue_count {
-                    let prefer_immediate = false; //address_book.provisioning_trackers.len()/4 > executing_queue_count;
-                    if let Some(ee) = Self::schedule_next_execution(&task_sender, runnable_queue, address_book, &mut contended_count, prefer_immediate, &sequence_time, &mut queue_clock, &mut execute_clock) {
+                while (executing_queue_count + provisioning_tracker_count) < max_executing_queue_count {
+                    let prefer_immediate = provisioning_tracker_count/4 > executing_queue_count;
+                    if let Some(ee) = Self::schedule_next_execution(&task_sender, runnable_queue, address_book, &mut contended_count, prefer_immediate, &sequence_time, &mut queue_clock, &mut execute_clock, &mut provisioning_tracker_count) {
                         executing_queue_count += 1;
                         to_execute_substage.send(ee).unwrap();
                     } else {
@@ -1079,7 +1084,7 @@ impl ScheduleStage {
                     if !empty_from_exec {
                         let mut processed_execution_environment = from_exec.recv().unwrap();
                         executing_queue_count -= 1;
-                        Self::commit_completed_execution(&mut processed_execution_environment, address_book, &mut execute_clock);
+                        Self::commit_completed_execution(&mut processed_execution_environment, address_book, &mut execute_clock, &mut provisioning_tracker_count);
                         to_next_stage.send(processed_execution_environment).unwrap();
                     } else {
                         from_exec_len = from_exec.len();

From 80908753f913db000c6b22db273b73cbbb59d7f9 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 27 Aug 2022 14:20:11 +0900
Subject: [PATCH 1514/3199] save

---
 scheduler/src/lib.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 3af1888120cfbe..e19157862e253c 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -797,7 +797,7 @@ impl ScheduleStage {
                 *contended_count = contended_count.checked_sub(1).unwrap();
                 next_task.mark_as_uncontended();
                 let tracker = std::sync::Arc::new(ProvisioningTracker::new(provisional_count, TaskInQueue::clone(task)));
-                provisioning_tracker_count += 1;
+                *provisioning_tracker_count += 1;
                 Self::finalize_lock_for_provisional_execution(
                     address_book,
                     next_task,
@@ -908,7 +908,7 @@ impl ScheduleStage {
                     if tracker.is_fulfilled() {
                         trace!("provisioning tracker progress: {} => {} (!)", tracker.prev_count(), tracker.count());
                         address_book.fulfilled_provisional_task_ids.insert(tracker.task.unique_weight, TaskInQueue::clone(&tracker.task));
-                        provisioning_tracker_count -= 1;
+                        *provisioning_tracker_count -= 1;
                     } else {
                         trace!("provisioning tracker progress: {} => {}", tracker.prev_count(), tracker.count());
                     }

From df3db9d7956056c1bf778e38c2714086445068e0 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 27 Aug 2022 14:21:45 +0900
Subject: [PATCH 1515/3199] save

---
 scheduler/src/lib.rs | 10 ++++------
 1 file changed, 4 insertions(+), 6 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index e19157862e253c..6010c62fe94893 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -796,13 +796,12 @@ impl ScheduleStage {
                 trace!("provisional exec: [{}/{}]", provisional_count, lock_count);
                 *contended_count = contended_count.checked_sub(1).unwrap();
                 next_task.mark_as_uncontended();
-                let tracker = std::sync::Arc::new(ProvisioningTracker::new(provisional_count, TaskInQueue::clone(task)));
+                let tracker = std::sync::Arc::new(ProvisioningTracker::new(provisional_count, TaskInQueue::clone(a2)));
                 *provisioning_tracker_count += 1;
                 Self::finalize_lock_for_provisional_execution(
                     address_book,
                     next_task,
-                    &a2,
-                    provisional_count,
+                    &tracker
                 );
 
                 return None;
@@ -830,13 +829,12 @@ impl ScheduleStage {
     fn finalize_lock_for_provisional_execution(
         address_book: &mut AddressBook,
         next_task: &mut Task,
-        task: &TaskInQueue,
-        provisional_count: usize,
+        tracker: &std::sync::Arc<ProvisioningTracker>,
     ) {
         for l in next_task.tx.1.iter_mut() {
             match l.status {
                 LockStatus::Provisional => {
-                    l.target.page_mut().provisional_task_ids.push(std::sync::Arc::clone(&tracker));
+                    l.target.page_mut().provisional_task_ids.push(std::sync::Arc::clone(tracker));
                 }
                 LockStatus::Succeded => {
                     // do nothing

From 277944ae55ec371ba4788bf9d09e3acb9f43e39c Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 27 Aug 2022 14:22:03 +0900
Subject: [PATCH 1516/3199] save

---
 scheduler/src/lib.rs | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 6010c62fe94893..bca7bc863b579e 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -796,12 +796,12 @@ impl ScheduleStage {
                 trace!("provisional exec: [{}/{}]", provisional_count, lock_count);
                 *contended_count = contended_count.checked_sub(1).unwrap();
                 next_task.mark_as_uncontended();
-                let tracker = std::sync::Arc::new(ProvisioningTracker::new(provisional_count, TaskInQueue::clone(a2)));
+                let tracker = std::sync::Arc::new(ProvisioningTracker::new(provisional_count, TaskInQueue::clone(&a2)));
                 *provisioning_tracker_count += 1;
                 Self::finalize_lock_for_provisional_execution(
                     address_book,
                     next_task,
-                    &tracker
+                    tracker
                 );
 
                 return None;
@@ -829,7 +829,7 @@ impl ScheduleStage {
     fn finalize_lock_for_provisional_execution(
         address_book: &mut AddressBook,
         next_task: &mut Task,
-        tracker: &std::sync::Arc<ProvisioningTracker>,
+        tracker: std::sync::Arc<ProvisioningTracker>,
     ) {
         for l in next_task.tx.1.iter_mut() {
             match l.status {

From 0bab18c05740c872f14a9e89e16e5305fefb9dbb Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 27 Aug 2022 14:22:13 +0900
Subject: [PATCH 1517/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index bca7bc863b579e..65bd7f12359e80 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -834,7 +834,7 @@ impl ScheduleStage {
         for l in next_task.tx.1.iter_mut() {
             match l.status {
                 LockStatus::Provisional => {
-                    l.target.page_mut().provisional_task_ids.push(std::sync::Arc::clone(tracker));
+                    l.target.page_mut().provisional_task_ids.push(std::sync::Arc::clone(&tracker));
                 }
                 LockStatus::Succeded => {
                     // do nothing

From 36e60cf5d4d682c4be73f48170a419fa2cbaaa86 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 27 Aug 2022 14:24:02 +0900
Subject: [PATCH 1518/3199] save

---
 scheduler/src/lib.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 65bd7f12359e80..b693e367d87f23 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -797,7 +797,7 @@ impl ScheduleStage {
                 *contended_count = contended_count.checked_sub(1).unwrap();
                 next_task.mark_as_uncontended();
                 let tracker = std::sync::Arc::new(ProvisioningTracker::new(provisional_count, TaskInQueue::clone(&a2)));
-                *provisioning_tracker_count += 1;
+                *provisioning_tracker_count = provisioning_tracker_count.checked_add(1).unwrap();
                 Self::finalize_lock_for_provisional_execution(
                     address_book,
                     next_task,
@@ -906,7 +906,7 @@ impl ScheduleStage {
                     if tracker.is_fulfilled() {
                         trace!("provisioning tracker progress: {} => {} (!)", tracker.prev_count(), tracker.count());
                         address_book.fulfilled_provisional_task_ids.insert(tracker.task.unique_weight, TaskInQueue::clone(&tracker.task));
-                        *provisioning_tracker_count -= 1;
+                        *provisioning_tracker_count = provisioning_tracker_count.checked_sub(1).unwrap();
                     } else {
                         trace!("provisioning tracker progress: {} => {}", tracker.prev_count(), tracker.count());
                     }

From 20fe2aaffa51269fc97f949caac9ce657eb39815 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 27 Aug 2022 14:26:34 +0900
Subject: [PATCH 1519/3199] save

---
 scheduler/src/lib.rs | 33 ++++-----------------------------
 1 file changed, 4 insertions(+), 29 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index b693e367d87f23..d62cbd59dbca65 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -632,34 +632,11 @@ impl ScheduleStage {
     fn push_to_runnable_queue(
         task: TaskInQueue,
         runnable_queue: &mut TaskQueue,
-        unique_key: &mut u64,
     ) {
-        // manage randomness properly for future scheduling determinism
-        //let mut rng = rand::thread_rng();
-
-        //let ix = 23;
-        //let tx = bank
-        //    .verify_transaction(
-        //        tx,
-        //        solana_sdk::transaction::TransactionVerificationMode::FullVerification,
-        //    )
-        //    .unwrap();
-        //tx.foo();
-        //let unique_weight = (weight << 32) | (*unique_key & 0x0000_0000_ffff_ffff);
-        let unique_weight = task.unique_weight;
-
         runnable_queue.add_to_schedule(
-            /*
-            UniqueWeight {
-                weight,
-                //unique_key: solana_sdk::hash::new_rand(&mut rng),
-                unique_key: *unique_key,
-            },
-            */
-            unique_weight,
+            task.unique_weight,
             task,
         );
-        *unique_key -= 1;
     }
 
     #[inline(never)]
@@ -982,12 +959,11 @@ impl ScheduleStage {
     fn register_runnable_task(
         weighted_tx: TaskInQueue,
         runnable_queue: &mut TaskQueue,
-        unique_key: &mut u64,
         sequence_time: &mut usize,
     ) {
         weighted_tx.record_sequence_time(*sequence_time);
         *sequence_time = sequence_time.checked_add(1).unwrap();
-        Self::push_to_runnable_queue(weighted_tx, runnable_queue, unique_key)
+        Self::push_to_runnable_queue(weighted_tx, runnable_queue)
     }
 
     pub fn run(
@@ -1000,7 +976,6 @@ impl ScheduleStage {
         maybe_to_next_stage: Option<&crossbeam_channel::Sender<Box<ExecutionEnvironment>>>, // assume nonblocking
     ) {
         let mut executing_queue_count = 0;
-        let mut current_unique_key = u64::max_value();
         let mut contended_count = 0;
         let mut provisioning_tracker_count = 0;
         let mut sequence_time = 0;
@@ -1054,7 +1029,7 @@ impl ScheduleStage {
                }
                recv(from) -> maybe_from => {
                    let task = maybe_from.unwrap();
-                   Self::register_runnable_task(task, runnable_queue, &mut current_unique_key, &mut sequence_time);
+                   Self::register_runnable_task(task, runnable_queue, &mut sequence_time);
                }
             }
 
@@ -1089,7 +1064,7 @@ impl ScheduleStage {
                     }
                     if !empty_from {
                        let task = from.recv().unwrap();
-                       Self::register_runnable_task(task, runnable_queue, &mut current_unique_key, &mut sequence_time);
+                       Self::register_runnable_task(task, runnable_queue, &mut sequence_time);
                     } else {
                         from_len = from.len();
                     }

From c0c221760461b56fdfad11c988fa1cae275adf12 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 27 Aug 2022 14:28:50 +0900
Subject: [PATCH 1520/3199] save

---
 scheduler/src/lib.rs | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index d62cbd59dbca65..d02263e7e1f1c2 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1023,7 +1023,7 @@ impl ScheduleStage {
             crossbeam_channel::select! {
                recv(from_exec) -> maybe_from_exec => {
                    let mut processed_execution_environment = maybe_from_exec.unwrap();
-                    executing_queue_count -= 1;
+                    executing_queue_count = executing_queue_count.checked_sub(1).unwrap();
                     Self::commit_completed_execution(&mut processed_execution_environment, address_book, &mut execute_clock, &mut provisioning_tracker_count);
                     to_next_stage.send(processed_execution_environment).unwrap();
                }
@@ -1040,7 +1040,7 @@ impl ScheduleStage {
                 while (executing_queue_count + provisioning_tracker_count) < max_executing_queue_count {
                     let prefer_immediate = provisioning_tracker_count/4 > executing_queue_count;
                     if let Some(ee) = Self::schedule_next_execution(&task_sender, runnable_queue, address_book, &mut contended_count, prefer_immediate, &sequence_time, &mut queue_clock, &mut execute_clock, &mut provisioning_tracker_count) {
-                        executing_queue_count += 1;
+                        executing_queue_count = executing_queue_count.checked_add(1).unwrap();
                         to_execute_substage.send(ee).unwrap();
                     } else {
                         break;
@@ -1056,7 +1056,7 @@ impl ScheduleStage {
                 } else {
                     if !empty_from_exec {
                         let mut processed_execution_environment = from_exec.recv().unwrap();
-                        executing_queue_count -= 1;
+                        executing_queue_count = executing_queue_count.checked_sub(1).unwrap();
                         Self::commit_completed_execution(&mut processed_execution_environment, address_book, &mut execute_clock, &mut provisioning_tracker_count);
                         to_next_stage.send(processed_execution_environment).unwrap();
                     } else {

From ab6cc895743a47b8a950b0ed41353213761d645d Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 27 Aug 2022 14:29:21 +0900
Subject: [PATCH 1521/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index d02263e7e1f1c2..ad7ac24b132281 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -975,7 +975,7 @@ impl ScheduleStage {
         to_execute_substage: &crossbeam_channel::Sender<Box<ExecutionEnvironment>>,
         maybe_to_next_stage: Option<&crossbeam_channel::Sender<Box<ExecutionEnvironment>>>, // assume nonblocking
     ) {
-        let mut executing_queue_count = 0;
+        let mut executing_queue_count = 0_usize;
         let mut contended_count = 0;
         let mut provisioning_tracker_count = 0;
         let mut sequence_time = 0;

From fc8b94d7856f20892f6926e8e0cb2137a9d3d5eb Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 27 Aug 2022 14:33:27 +0900
Subject: [PATCH 1522/3199] save

---
 scheduler/src/lib.rs | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index ad7ac24b132281..485942021a3e59 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1018,8 +1018,6 @@ impl ScheduleStage {
         }
 
         loop {
-            trace!("schedule_once (from: {}, to: {}, runnnable: {}, contended: {}, (immediate+provisional)/max: ({}+{})/{}) active from contended: {}!", from.len(), to_execute_substage.len(), runnable_queue.task_count(), contended_count, executing_queue_count, provisioning_tracker_count, max_executing_queue_count, address_book.uncontended_task_ids.len());
-
             crossbeam_channel::select! {
                recv(from_exec) -> maybe_from_exec => {
                    let mut processed_execution_environment = maybe_from_exec.unwrap();
@@ -1038,6 +1036,7 @@ impl ScheduleStage {
 
             loop {
                 while (executing_queue_count + provisioning_tracker_count) < max_executing_queue_count {
+                    trace!("schedule_once (from: {}, to: {}, runnnable: {}, contended: {}, (immediate+provisional)/max: ({}+{})/{}) active from contended: {}!", from.len(), to_execute_substage.len(), runnable_queue.task_count(), contended_count, executing_queue_count, provisioning_tracker_count, max_executing_queue_count, address_book.uncontended_task_ids.len());
                     let prefer_immediate = provisioning_tracker_count/4 > executing_queue_count;
                     if let Some(ee) = Self::schedule_next_execution(&task_sender, runnable_queue, address_book, &mut contended_count, prefer_immediate, &sequence_time, &mut queue_clock, &mut execute_clock, &mut provisioning_tracker_count) {
                         executing_queue_count = executing_queue_count.checked_add(1).unwrap();

From 31292f6a99fb2942a19c0c3bda16b8f8f3adb332 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 28 Aug 2022 15:57:28 +0900
Subject: [PATCH 1523/3199] save

---
 scheduler/src/lib.rs | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 485942021a3e59..8b24d64b3ff553 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1062,8 +1062,11 @@ impl ScheduleStage {
                         from_exec_len = from_exec.len();
                     }
                     if !empty_from {
-                       let task = from.recv().unwrap();
-                       Self::register_runnable_task(task, runnable_queue, &mut sequence_time);
+                       let task = from.recv();
+                       if task.is_err() {
+                            error!("odd: {} {:?}", from_len, task);
+                        }
+                       Self::register_runnable_task(task.unwrap(), runnable_queue, &mut sequence_time);
                     } else {
                         from_len = from.len();
                     }

From 9ddaceb3e73e183190923727fea3cf6f13b2845a Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 28 Aug 2022 15:59:50 +0900
Subject: [PATCH 1524/3199] save

---
 transaction-scheduler-bench/src/main.rs | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/transaction-scheduler-bench/src/main.rs b/transaction-scheduler-bench/src/main.rs
index 72ed0b4f2ce537..0c9bca75a90d71 100644
--- a/transaction-scheduler-bench/src/main.rs
+++ b/transaction-scheduler-bench/src/main.rs
@@ -499,6 +499,9 @@ fn send_packets(
         }
         if start.elapsed() > duration {
             info!("stopping packet sending");
+            while packet_batch_sender.len() > 0 {
+                std::thread::sleep(Duration::from_micros(10));
+            }
             break;
         }
         let (packet_batches, packet_build_time) = measure!(build_packet_batches(

From 8c34b63fdea364029a46b05e8de7320f2504a703 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 28 Aug 2022 16:00:46 +0900
Subject: [PATCH 1525/3199] save

---
 transaction-scheduler-bench/src/main.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/transaction-scheduler-bench/src/main.rs b/transaction-scheduler-bench/src/main.rs
index 0c9bca75a90d71..456638e44fbf23 100644
--- a/transaction-scheduler-bench/src/main.rs
+++ b/transaction-scheduler-bench/src/main.rs
@@ -500,7 +500,7 @@ fn send_packets(
         if start.elapsed() > duration {
             info!("stopping packet sending");
             while packet_batch_sender.len() > 0 {
-                std::thread::sleep(Duration::from_micros(10));
+                std::thread::sleep(Duration::from_millis(10));
             }
             break;
         }

From cdcd565cac07b1e8bd4b6ebd43317e4a2eca3c72 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 28 Aug 2022 16:02:08 +0900
Subject: [PATCH 1526/3199] save

---
 scheduler/src/lib.rs | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 8b24d64b3ff553..edcb264b83c965 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1062,9 +1062,10 @@ impl ScheduleStage {
                         from_exec_len = from_exec.len();
                     }
                     if !empty_from {
+                        let old_len = from.len();
                        let task = from.recv();
                        if task.is_err() {
-                            error!("odd: {} {:?}", from_len, task);
+                            error!("odd: {} {:?} {} -> {}", from_len, task, old_len, from.len());
                         }
                        Self::register_runnable_task(task.unwrap(), runnable_queue, &mut sequence_time);
                     } else {

From 29daf099fd39e792564cf323393ab77bfd971513 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 28 Aug 2022 16:04:46 +0900
Subject: [PATCH 1527/3199] save

---
 scheduler/src/lib.rs | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index edcb264b83c965..b47af5638c7498 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1058,6 +1058,7 @@ impl ScheduleStage {
                         executing_queue_count = executing_queue_count.checked_sub(1).unwrap();
                         Self::commit_completed_execution(&mut processed_execution_environment, address_book, &mut execute_clock, &mut provisioning_tracker_count);
                         to_next_stage.send(processed_execution_environment).unwrap();
+                        from_exec_len -= 1;
                     } else {
                         from_exec_len = from_exec.len();
                     }
@@ -1067,6 +1068,7 @@ impl ScheduleStage {
                        if task.is_err() {
                             error!("odd: {} {:?} {} -> {}", from_len, task, old_len, from.len());
                         }
+                       from_len -= 1;
                        Self::register_runnable_task(task.unwrap(), runnable_queue, &mut sequence_time);
                     } else {
                         from_len = from.len();

From 98ce79086a9fb63bf185cd98c26156b92e4b7d93 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 28 Aug 2022 16:06:35 +0900
Subject: [PATCH 1528/3199] save

---
 scheduler/src/lib.rs | 11 ++++-------
 1 file changed, 4 insertions(+), 7 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index b47af5638c7498..ea65ad2c102cd4 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1055,21 +1055,18 @@ impl ScheduleStage {
                 } else {
                     if !empty_from_exec {
                         let mut processed_execution_environment = from_exec.recv().unwrap();
+                        from_exec_len -= 1;
+
                         executing_queue_count = executing_queue_count.checked_sub(1).unwrap();
                         Self::commit_completed_execution(&mut processed_execution_environment, address_book, &mut execute_clock, &mut provisioning_tracker_count);
                         to_next_stage.send(processed_execution_environment).unwrap();
-                        from_exec_len -= 1;
                     } else {
                         from_exec_len = from_exec.len();
                     }
                     if !empty_from {
-                        let old_len = from.len();
-                       let task = from.recv();
-                       if task.is_err() {
-                            error!("odd: {} {:?} {} -> {}", from_len, task, old_len, from.len());
-                        }
+                       let task = from.recv().unwrap();
                        from_len -= 1;
-                       Self::register_runnable_task(task.unwrap(), runnable_queue, &mut sequence_time);
+                       Self::register_runnable_task(task, runnable_queue, &mut sequence_time);
                     } else {
                         from_len = from.len();
                     }

From fd8597613220fe58249976c96dc305821223d48f Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 28 Aug 2022 21:11:27 +0900
Subject: [PATCH 1529/3199] save

---
 scheduler/src/lib.rs | 16 ++++++++++++++++
 1 file changed, 16 insertions(+)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index ea65ad2c102cd4..c5934fe441d51a 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1020,12 +1020,28 @@ impl ScheduleStage {
         loop {
             crossbeam_channel::select! {
                recv(from_exec) -> maybe_from_exec => {
+                   if maybe_from_exec.is_err() {
+                       assert_eq!(from_exec.len(), 0);
+                       if from.len() == 0 {
+                           break;
+                       } else {
+                           continue;
+                       }
+                   }
                    let mut processed_execution_environment = maybe_from_exec.unwrap();
                     executing_queue_count = executing_queue_count.checked_sub(1).unwrap();
                     Self::commit_completed_execution(&mut processed_execution_environment, address_book, &mut execute_clock, &mut provisioning_tracker_count);
                     to_next_stage.send(processed_execution_environment).unwrap();
                }
                recv(from) -> maybe_from => {
+                   if maybe_from.is_err() {
+                       assert_eq!(from.len(), 0);
+                       if from_exec.len() == 0 {
+                           break;
+                       } else {
+                           continue;
+                       }
+                   }
                    let task = maybe_from.unwrap();
                    Self::register_runnable_task(task, runnable_queue, &mut sequence_time);
                }

From b742f8f691048fba5238bbe151cbd01537c7ca39 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 28 Aug 2022 21:13:06 +0900
Subject: [PATCH 1530/3199] save

---
 scheduler/src/lib.rs | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index c5934fe441d51a..9b3bd98e4a1e60 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -987,13 +987,14 @@ impl ScheduleStage {
             (to_next_stage, None)
         } else {
             let h = std::thread::Builder::new().name("sol-reaper".to_string()).spawn(move || {
-                while let mut a = ee_receiver.recv().unwrap() {
+                while let Ok(mut a) = ee_receiver.recv() {
                     assert!(a.task.tx.1.is_empty());
                     assert!(a.task.sequence_time() != usize::max_value());
                     //let lock_attempts = std::mem::take(&mut a.lock_attempts);
                     //drop(lock_attempts);
                     //TaskInQueue::get_mut(&mut a.task).unwrap();
                 }
+                assert_eq!(ee_receiver.len(), 0);
             }).unwrap();
 
             (&ee_sender, Some(h))

From ce9cad2ce9a4f7bca0912964a9fbcb389afe855b Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 28 Aug 2022 21:14:03 +0900
Subject: [PATCH 1531/3199] save

---
 scheduler/src/lib.rs | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 9b3bd98e4a1e60..523c1b818977d7 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1007,7 +1007,7 @@ impl ScheduleStage {
         for thx in 0..indexer_count {
             let task_receiver = task_receiver.clone();
             let h = std::thread::Builder::new().name(format!("sol-indexer{:02}", thx)).spawn(move || {
-                while let (task, ll) = task_receiver.recv().unwrap() {
+                while let Ok((task, ll)) = task_receiver.recv() {
                     for lock_attempt in ll {
                         if task.already_finished() {
                             break;
@@ -1015,6 +1015,7 @@ impl ScheduleStage {
                         lock_attempt.contended_unique_weights().insert_task(task.unique_weight, TaskInQueue::clone(&task));
                     }
                 }
+                assert_eq!(task_receiver.len(), 0);
             }).unwrap();
         }
 

From b78721b78b96dacd9ed879d30fdc4598a370aa02 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 28 Aug 2022 21:16:22 +0900
Subject: [PATCH 1532/3199] save

---
 scheduler/src/lib.rs | 12 ++++++++----
 1 file changed, 8 insertions(+), 4 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 523c1b818977d7..2bb144dba6ee05 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1066,8 +1066,16 @@ impl ScheduleStage {
                 if first_iteration {
                     first_iteration = false;
                     (from_len,  from_exec_len) = (from.len(), from_exec.len());
+                } else {
+                    if empty_from {
+                        from_len = from.len();
+                    }
+                    if empty_from_exec {
+                        from_exec_len = from_exec.len();
+                    }
                 }
                 let (empty_from, empty_from_exec) = (from_len == 0, from_exec_len == 0);
+
                 if empty_from && empty_from_exec {
                    break;
                 } else {
@@ -1078,15 +1086,11 @@ impl ScheduleStage {
                         executing_queue_count = executing_queue_count.checked_sub(1).unwrap();
                         Self::commit_completed_execution(&mut processed_execution_environment, address_book, &mut execute_clock, &mut provisioning_tracker_count);
                         to_next_stage.send(processed_execution_environment).unwrap();
-                    } else {
-                        from_exec_len = from_exec.len();
                     }
                     if !empty_from {
                        let task = from.recv().unwrap();
                        from_len -= 1;
                        Self::register_runnable_task(task, runnable_queue, &mut sequence_time);
-                    } else {
-                        from_len = from.len();
                     }
                 }
             }

From eabffa27e4bd1ca53b69cdc88616d022ca833139 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 28 Aug 2022 21:17:31 +0900
Subject: [PATCH 1533/3199] save

---
 scheduler/src/lib.rs | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 2bb144dba6ee05..2ddf17bd45b061 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1050,6 +1050,7 @@ impl ScheduleStage {
             }
 
             let mut first_iteration = true;
+            let mut (empty_from, empty_from_exec) = (false, false);
             let (mut from_len, mut from_exec_len) = (0, 0);
 
             loop {
@@ -1074,7 +1075,7 @@ impl ScheduleStage {
                         from_exec_len = from_exec.len();
                     }
                 }
-                let (empty_from, empty_from_exec) = (from_len == 0, from_exec_len == 0);
+                (empty_from, empty_from_exec) = (from_len == 0, from_exec_len == 0);
 
                 if empty_from && empty_from_exec {
                    break;

From 7320fcaf16b640885d2405700305175cf8816dd1 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 28 Aug 2022 21:17:43 +0900
Subject: [PATCH 1534/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 2ddf17bd45b061..d09712347f1fb0 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1050,7 +1050,7 @@ impl ScheduleStage {
             }
 
             let mut first_iteration = true;
-            let mut (empty_from, empty_from_exec) = (false, false);
+            let (mut empty_from, mut empty_from_exec) = (false, false);
             let (mut from_len, mut from_exec_len) = (0, 0);
 
             loop {

From b0e92afefbfa75e479b417bc192b93ab81b1ad49 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 28 Aug 2022 21:19:26 +0900
Subject: [PATCH 1535/3199] save

---
 scheduler/src/lib.rs | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index d09712347f1fb0..610e8fb4477a9a 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1082,15 +1082,14 @@ impl ScheduleStage {
                 } else {
                     if !empty_from_exec {
                         let mut processed_execution_environment = from_exec.recv().unwrap();
-                        from_exec_len -= 1;
-
+                        from_exec_len = from_exec_len.checked_sub(1).unwrap();
                         executing_queue_count = executing_queue_count.checked_sub(1).unwrap();
                         Self::commit_completed_execution(&mut processed_execution_environment, address_book, &mut execute_clock, &mut provisioning_tracker_count);
                         to_next_stage.send(processed_execution_environment).unwrap();
                     }
                     if !empty_from {
                        let task = from.recv().unwrap();
-                       from_len -= 1;
+                       from_len = from_len.checked_sub(1).unwrap();
                        Self::register_runnable_task(task, runnable_queue, &mut sequence_time);
                     }
                 }

From d783f995da07b37a38a9d19ea659b017c31fb401 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 28 Aug 2022 21:29:40 +0900
Subject: [PATCH 1536/3199] save

---
 scheduler/src/lib.rs | 1 +
 1 file changed, 1 insertion(+)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 610e8fb4477a9a..23fa6774974e61 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1095,6 +1095,7 @@ impl ScheduleStage {
                 }
             }
         }
+        info!("run finished...");
     }
 }
 

From 31d3e586c60c8ede93239059a228472344a2e956 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 28 Aug 2022 21:30:14 +0900
Subject: [PATCH 1537/3199] save

---
 transaction-scheduler-bench/src/main.rs | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/transaction-scheduler-bench/src/main.rs b/transaction-scheduler-bench/src/main.rs
index 456638e44fbf23..72ed0b4f2ce537 100644
--- a/transaction-scheduler-bench/src/main.rs
+++ b/transaction-scheduler-bench/src/main.rs
@@ -499,9 +499,6 @@ fn send_packets(
         }
         if start.elapsed() > duration {
             info!("stopping packet sending");
-            while packet_batch_sender.len() > 0 {
-                std::thread::sleep(Duration::from_millis(10));
-            }
             break;
         }
         let (packet_batches, packet_build_time) = measure!(build_packet_batches(

From 1687127580dea05dae62fb2e8698a3aea58da053 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 28 Aug 2022 21:31:58 +0900
Subject: [PATCH 1538/3199] save

---
 scheduler/src/lib.rs | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 23fa6774974e61..d7c7d78817e145 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -751,10 +751,10 @@ impl ScheduleStage {
                     trace!("move to contended due to lock failure [{}/{}/{}]", unlockable_count, provisional_count, lock_count);
                     next_task.mark_as_contended();
                     *contended_count = contended_count.checked_add(1).unwrap();
-                    //for lock_attempt in next_task.tx.1.iter() {
-                    //    lock_attempt.contended_unique_weights().insert_task(unique_weight, TaskInQueue::clone(&a2));
-                    //}
-                    task_sender.send((TaskInQueue::clone(&a2), std::mem::take(&mut next_task.for_indexer))).unwrap();
+                    for lock_attempt in next_task.tx.1.iter() {
+                        lock_attempt.contended_unique_weights().insert_task(unique_weight, TaskInQueue::clone(&a2));
+                    }
+                    //task_sender.send((TaskInQueue::clone(&a2), std::mem::take(&mut next_task.for_indexer))).unwrap();
                     // maybe run lightweight prune logic on contended_queue here.
                 } else {
                     trace!("relock failed [{}/{}/{}]; remains in contended: {:?} contention: {}", unlockable_count, provisional_count, lock_count, &unique_weight, next_task.contention_count);

From 43cc973d5d65c6951d299a9ca72c0aa12a52b78f Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 28 Aug 2022 21:32:44 +0900
Subject: [PATCH 1539/3199] save

---
 scheduler/src/lib.rs | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index d7c7d78817e145..23fa6774974e61 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -751,10 +751,10 @@ impl ScheduleStage {
                     trace!("move to contended due to lock failure [{}/{}/{}]", unlockable_count, provisional_count, lock_count);
                     next_task.mark_as_contended();
                     *contended_count = contended_count.checked_add(1).unwrap();
-                    for lock_attempt in next_task.tx.1.iter() {
-                        lock_attempt.contended_unique_weights().insert_task(unique_weight, TaskInQueue::clone(&a2));
-                    }
-                    //task_sender.send((TaskInQueue::clone(&a2), std::mem::take(&mut next_task.for_indexer))).unwrap();
+                    //for lock_attempt in next_task.tx.1.iter() {
+                    //    lock_attempt.contended_unique_weights().insert_task(unique_weight, TaskInQueue::clone(&a2));
+                    //}
+                    task_sender.send((TaskInQueue::clone(&a2), std::mem::take(&mut next_task.for_indexer))).unwrap();
                     // maybe run lightweight prune logic on contended_queue here.
                 } else {
                     trace!("relock failed [{}/{}/{}]; remains in contended: {:?} contention: {}", unlockable_count, provisional_count, lock_count, &unique_weight, next_task.contention_count);

From 88189cb0bb6fe6d46b9a6f2cef5c7a8930cc1b38 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 28 Aug 2022 21:39:39 +0900
Subject: [PATCH 1540/3199] save

---
 transaction-scheduler-bench/src/main.rs | 1 +
 1 file changed, 1 insertion(+)

diff --git a/transaction-scheduler-bench/src/main.rs b/transaction-scheduler-bench/src/main.rs
index 72ed0b4f2ce537..4e0566e5814760 100644
--- a/transaction-scheduler-bench/src/main.rs
+++ b/transaction-scheduler-bench/src/main.rs
@@ -308,6 +308,7 @@ fn execution_worker(
 ) {
     loop {
         if exit.load(Ordering::Relaxed) {
+            info("executor exit!");
             break;
         }
 

From 55b9094c2491e59ab65238aa61c9314fd9228266 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 28 Aug 2022 21:39:48 +0900
Subject: [PATCH 1541/3199] save

---
 transaction-scheduler-bench/src/main.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/transaction-scheduler-bench/src/main.rs b/transaction-scheduler-bench/src/main.rs
index 4e0566e5814760..f6dd4f6c9d8bca 100644
--- a/transaction-scheduler-bench/src/main.rs
+++ b/transaction-scheduler-bench/src/main.rs
@@ -308,7 +308,7 @@ fn execution_worker(
 ) {
     loop {
         if exit.load(Ordering::Relaxed) {
-            info("executor exit!");
+            info!("executor exit!");
             break;
         }
 

From 91265a7aba071d42be946eb7174d8abaade0fb8c Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 28 Aug 2022 21:42:51 +0900
Subject: [PATCH 1542/3199] save

---
 transaction-scheduler-bench/src/main.rs | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/transaction-scheduler-bench/src/main.rs b/transaction-scheduler-bench/src/main.rs
index f6dd4f6c9d8bca..af3394a66da1ef 100644
--- a/transaction-scheduler-bench/src/main.rs
+++ b/transaction-scheduler-bench/src/main.rs
@@ -309,6 +309,9 @@ fn execution_worker(
     loop {
         if exit.load(Ordering::Relaxed) {
             info!("executor exit!");
+            while completed_transaction_sender.len() > 0 {
+                std::thread::sleep(Duration::from_millis(100));
+            }
             break;
         }
 

From 8c9448fa3582b575b9d8865cf9ae8b35f7b2bae5 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 28 Aug 2022 21:45:19 +0900
Subject: [PATCH 1543/3199] save

---
 transaction-scheduler-bench/src/main.rs | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/transaction-scheduler-bench/src/main.rs b/transaction-scheduler-bench/src/main.rs
index af3394a66da1ef..343e1b4378af32 100644
--- a/transaction-scheduler-bench/src/main.rs
+++ b/transaction-scheduler-bench/src/main.rs
@@ -309,9 +309,7 @@ fn execution_worker(
     loop {
         if exit.load(Ordering::Relaxed) {
             info!("executor exit!");
-            while completed_transaction_sender.len() > 0 {
-                std::thread::sleep(Duration::from_millis(100));
-            }
+            std::thread::sleep(Duration::from_millis(1000));
             break;
         }
 

From 5a419d39b182cdd8d255ff5c02e352af8064f845 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 28 Aug 2022 21:50:22 +0900
Subject: [PATCH 1544/3199] save

---
 scheduler/src/lib.rs                    | 42 +++++++++++++++++++++++++
 transaction-scheduler-bench/src/main.rs | 41 +-----------------------
 2 files changed, 43 insertions(+), 40 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 23fa6774974e61..f5ef2ff90aa65d 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -47,6 +47,48 @@ impl ExecutionEnvironment {
     //fn abort() {
     //  pass AtomicBool into InvokeContext??
     //}
+    //
+    fn reindex(&self) {
+        let uq = self.unique_weight;
+        //self.task.trace_timestamps("in_exec(self)");
+        let should_remove = self.task.contention_count > 0;
+        for mut lock_attempt in self.lock_attempts.iter_mut() {
+            let contended_unique_weights = lock_attempt.contended_unique_weights();
+            contended_unique_weights.heaviest_task_cursor().map(|mut task_cursor| {
+                let mut found = true;
+                let mut removed = false;
+                let mut task = task_cursor.value();
+                //task.trace_timestamps("in_exec(initial list)");
+                while !task.currently_contended() {
+                    if task_cursor.key() == &uq {
+                        assert!(should_remove);
+                        removed = task_cursor.remove();
+                        assert!(removed);
+                    }
+                    if task.already_finished() {
+                        task_cursor.remove();
+                    }
+                    if let Some(new_cursor) = task_cursor.prev() {
+                        assert!(new_cursor.key() < task_cursor.key());
+                        task_cursor = new_cursor;
+                        task = task_cursor.value();
+                        //task.trace_timestamps("in_exec(subsequent list)");
+                    } else {
+                        found = false;
+                        break;
+                    }
+                }
+                if should_remove && !removed {
+                    contended_unique_weights.remove_task(&uq);
+                }
+                found.then(|| solana_scheduler::TaskInQueue::clone(task))
+            }).flatten().map(|task| {
+                //task.trace_timestamps(&format!("in_exec(heaviest:{})", self.task.queue_time_label()));
+                lock_attempt.heaviest_uncontended = Some(task);
+                ()
+            });
+        }
+    }
 }
 
 impl PageRc {
diff --git a/transaction-scheduler-bench/src/main.rs b/transaction-scheduler-bench/src/main.rs
index 343e1b4378af32..0a6f882767faea 100644
--- a/transaction-scheduler-bench/src/main.rs
+++ b/transaction-scheduler-bench/src/main.rs
@@ -360,46 +360,7 @@ fn handle_transaction_batch(
         .priority_collected
         .fetch_add(priority_collected, Ordering::Relaxed);
 
-    let uq = transaction_batch.unique_weight;
-    //transaction_batch.task.trace_timestamps("in_exec(self)");
-    let should_remove = transaction_batch.task.contention_count > 0;
-    for mut lock_attempt in transaction_batch.lock_attempts.iter_mut() {
-        let contended_unique_weights = lock_attempt.contended_unique_weights();
-        contended_unique_weights.heaviest_task_cursor().map(|mut task_cursor| {
-            let mut found = true;
-            let mut removed = false;
-            let mut task = task_cursor.value();
-            //task.trace_timestamps("in_exec(initial list)");
-            while !task.currently_contended() {
-                if task_cursor.key() == &uq {
-                    assert!(should_remove);
-                    removed = task_cursor.remove();
-                    assert!(removed);
-                }
-                if task.already_finished() {
-                    task_cursor.remove();
-                }
-                if let Some(new_cursor) = task_cursor.prev() {
-                    assert!(new_cursor.key() < task_cursor.key());
-                    task_cursor = new_cursor;
-                    task = task_cursor.value();
-                    //task.trace_timestamps("in_exec(subsequent list)");
-                } else {
-                    found = false;
-                    break;
-                }
-            }
-            if should_remove && !removed {
-                contended_unique_weights.remove_task(&uq);
-            }
-            found.then(|| solana_scheduler::TaskInQueue::clone(task))
-        }).flatten().map(|task| {
-            //task.trace_timestamps(&format!("in_exec(heaviest:{})", transaction_batch.task.queue_time_label()));
-            lock_attempt.heaviest_uncontended = Some(task);
-            ()
-        });
-    }
-    //error!("send from execute substage: {:?} seq: {}", uq, transaction_batch.task.sequence_time());
+    transaction_batch.reindex();
     completed_transaction_sender
         .send(transaction_batch)
         .unwrap();

From 4c689e63eff834344e1a5e3df9edbd8708e0ba92 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 28 Aug 2022 21:50:38 +0900
Subject: [PATCH 1545/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index f5ef2ff90aa65d..52a17f985edc77 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -81,7 +81,7 @@ impl ExecutionEnvironment {
                 if should_remove && !removed {
                     contended_unique_weights.remove_task(&uq);
                 }
-                found.then(|| solana_scheduler::TaskInQueue::clone(task))
+                found.then(|| TaskInQueue::clone(task))
             }).flatten().map(|task| {
                 //task.trace_timestamps(&format!("in_exec(heaviest:{})", self.task.queue_time_label()));
                 lock_attempt.heaviest_uncontended = Some(task);

From 66fcaeeeb3411ee9587b3d00aba41d8f1a936425 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 28 Aug 2022 21:50:51 +0900
Subject: [PATCH 1546/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 52a17f985edc77..68b1ccc9d28d3a 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -48,7 +48,7 @@ impl ExecutionEnvironment {
     //  pass AtomicBool into InvokeContext??
     //}
     //
-    fn reindex(&self) {
+    fn reindex(&mut self) {
         let uq = self.unique_weight;
         //self.task.trace_timestamps("in_exec(self)");
         let should_remove = self.task.contention_count > 0;

From e279b56fb73493512902df32823839da1795ef02 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 28 Aug 2022 21:51:01 +0900
Subject: [PATCH 1547/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 68b1ccc9d28d3a..7b2ebdb2ee7f16 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -48,7 +48,7 @@ impl ExecutionEnvironment {
     //  pass AtomicBool into InvokeContext??
     //}
     //
-    fn reindex(&mut self) {
+    pub fn reindex(&mut self) {
         let uq = self.unique_weight;
         //self.task.trace_timestamps("in_exec(self)");
         let should_remove = self.task.contention_count > 0;

From cacf6d47a77599fe7d6610200d9c26819e088926 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 28 Aug 2022 21:52:13 +0900
Subject: [PATCH 1548/3199] save

---
 scheduler/src/lib.rs | 1 +
 1 file changed, 1 insertion(+)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 7b2ebdb2ee7f16..c95583054c88c2 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -48,6 +48,7 @@ impl ExecutionEnvironment {
     //  pass AtomicBool into InvokeContext??
     //}
     //
+    #[inline(never)]
     pub fn reindex(&mut self) {
         let uq = self.unique_weight;
         //self.task.trace_timestamps("in_exec(self)");

From 73b6bc0da06e8e0c707059b058502d26e017757c Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 28 Aug 2022 21:53:02 +0900
Subject: [PATCH 1549/3199] save

---
 transaction-scheduler-bench/src/main.rs | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/transaction-scheduler-bench/src/main.rs b/transaction-scheduler-bench/src/main.rs
index 0a6f882767faea..61f814f7e0dff0 100644
--- a/transaction-scheduler-bench/src/main.rs
+++ b/transaction-scheduler-bench/src/main.rs
@@ -484,9 +484,6 @@ fn send_packets(
                 let uw = unique_weight.fetch_sub(1, std::sync::atomic::Ordering::SeqCst);
                 let p = (p << 32) | (uw & 0x0000_0000_ffff_ffff);
                 let t = solana_scheduler::Task::new_for_queue(p, v);
-                //for lock_attempt in t.tx.1.iter() {
-                //    lock_attempt.contended_unique_weights().insert_task(p, solana_scheduler::TaskInQueue::clone(&t));
-                //}
                 packet_batch_sender.send(t).unwrap();
             }
         }

From e97304637b969c50841f8e6fa0ca21c751fd9d26 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 28 Aug 2022 21:53:38 +0900
Subject: [PATCH 1550/3199] save

---
 scheduler/src/lib.rs                    | 1 +
 transaction-scheduler-bench/src/main.rs | 2 +-
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index c95583054c88c2..5545a3468b29c2 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -963,6 +963,7 @@ impl ScheduleStage {
     fn commit_completed_execution(ee: &mut ExecutionEnvironment, address_book: &mut AddressBook, commit_time: &mut usize, provisioning_tracker_count: &mut usize) {
         // do par()-ly?
 
+        ee.reindex();
         ee.task.record_commit_time(*commit_time);
         ee.task.trace_timestamps("commit");
         //*commit_time = commit_time.checked_add(1).unwrap();
diff --git a/transaction-scheduler-bench/src/main.rs b/transaction-scheduler-bench/src/main.rs
index 61f814f7e0dff0..6658c5010c07b8 100644
--- a/transaction-scheduler-bench/src/main.rs
+++ b/transaction-scheduler-bench/src/main.rs
@@ -360,7 +360,7 @@ fn handle_transaction_batch(
         .priority_collected
         .fetch_add(priority_collected, Ordering::Relaxed);
 
-    transaction_batch.reindex();
+    //transaction_batch.reindex();
     completed_transaction_sender
         .send(transaction_batch)
         .unwrap();

From 1999399c4eaa385b0f572403b0e7124f5cc7c395 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 28 Aug 2022 21:54:28 +0900
Subject: [PATCH 1551/3199] save

---
 scheduler/src/lib.rs | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 5545a3468b29c2..2a01001a737cad 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -794,10 +794,11 @@ impl ScheduleStage {
                     trace!("move to contended due to lock failure [{}/{}/{}]", unlockable_count, provisional_count, lock_count);
                     next_task.mark_as_contended();
                     *contended_count = contended_count.checked_add(1).unwrap();
-                    //for lock_attempt in next_task.tx.1.iter() {
-                    //    lock_attempt.contended_unique_weights().insert_task(unique_weight, TaskInQueue::clone(&a2));
-                    //}
-                    task_sender.send((TaskInQueue::clone(&a2), std::mem::take(&mut next_task.for_indexer))).unwrap();
+                    for lock_attempt in next_task.tx.1.iter() {
+                        lock_attempt.contended_unique_weights().insert_task(unique_weight, TaskInQueue::clone(&a2));
+                    }
+                    //task_sender.send((TaskInQueue::clone(&a2), std::mem::take(&mut next_task.for_indexer))).unwrap();
+
                     // maybe run lightweight prune logic on contended_queue here.
                 } else {
                     trace!("relock failed [{}/{}/{}]; remains in contended: {:?} contention: {}", unlockable_count, provisional_count, lock_count, &unique_weight, next_task.contention_count);

From 0793a89e1ac0afff3e10742a0ee69521e603a3f5 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 28 Aug 2022 21:55:45 +0900
Subject: [PATCH 1552/3199] save

---
 scheduler/src/lib.rs | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 2a01001a737cad..b759248acb82d7 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -890,11 +890,11 @@ impl ScheduleStage {
                 let mut inserted = false;
 
                 if let Some(task) = l.heaviest_uncontended.take() {
-                    //assert!(!task.already_finished());
-                    if task.currently_contended() {
+                    assert!(!task.already_finished());
+                    assert!(task.currently_contended())
                         inserted = true;
                         address_book.uncontended_task_ids.insert(task.unique_weight, task);
-                    } /*else {
+                    /*} /*else {
                         let contended_unique_weights = &page.contended_unique_weights;
                         contended_unique_weights.heaviest_task_cursor().map(|mut task_cursor| {
                             let mut found = true;

From 8d8029174e64facfbd75c09c29429630513a5412 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 28 Aug 2022 21:56:00 +0900
Subject: [PATCH 1553/3199] save

---
 scheduler/src/lib.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index b759248acb82d7..988f098fab76ab 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -891,10 +891,10 @@ impl ScheduleStage {
 
                 if let Some(task) = l.heaviest_uncontended.take() {
                     assert!(!task.already_finished());
-                    assert!(task.currently_contended())
+                    if task.currently_contended() {
                         inserted = true;
                         address_book.uncontended_task_ids.insert(task.unique_weight, task);
-                    /*} /*else {
+                    } /*else {
                         let contended_unique_weights = &page.contended_unique_weights;
                         contended_unique_weights.heaviest_task_cursor().map(|mut task_cursor| {
                             let mut found = true;

From 5bc7c29ac512c5b02323b6016e640dafa9d3c0aa Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 28 Aug 2022 21:57:58 +0900
Subject: [PATCH 1554/3199] save

---
 scheduler/src/lib.rs | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 988f098fab76ab..cdb9d7242aacf0 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -891,7 +891,8 @@ impl ScheduleStage {
 
                 if let Some(task) = l.heaviest_uncontended.take() {
                     assert!(!task.already_finished());
-                    if task.currently_contended() {
+                    if true || task.currently_contended() {
+                        assert!(task.currently_contended());
                         inserted = true;
                         address_book.uncontended_task_ids.insert(task.unique_weight, task);
                     } /*else {

From 6681dfb666b188deb9316981be09016331bec279 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 28 Aug 2022 21:59:18 +0900
Subject: [PATCH 1555/3199] save

---
 scheduler/src/lib.rs | 1 +
 1 file changed, 1 insertion(+)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index cdb9d7242aacf0..d927f655c5955d 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1110,6 +1110,7 @@ impl ScheduleStage {
                         break;
                     }
                 }
+                continue;
                 if first_iteration {
                     first_iteration = false;
                     (from_len,  from_exec_len) = (from.len(), from_exec.len());

From b1a4b06bb54bc0128a5e6ef22094307b78948573 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 28 Aug 2022 21:59:50 +0900
Subject: [PATCH 1556/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index d927f655c5955d..75e588f69e8e2e 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1110,7 +1110,7 @@ impl ScheduleStage {
                         break;
                     }
                 }
-                continue;
+                break;
                 if first_iteration {
                     first_iteration = false;
                     (from_len,  from_exec_len) = (from.len(), from_exec.len());

From bbe1415ad80e29601ec940bad3ce3549caf0aff1 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 28 Aug 2022 22:01:06 +0900
Subject: [PATCH 1557/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 75e588f69e8e2e..de1d1758433f54 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1110,7 +1110,7 @@ impl ScheduleStage {
                         break;
                     }
                 }
-                break;
+                //break;
                 if first_iteration {
                     first_iteration = false;
                     (from_len,  from_exec_len) = (from.len(), from_exec.len());

From bc5310567c5c151b180d360c60774eeb4088359e Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 28 Aug 2022 22:06:00 +0900
Subject: [PATCH 1558/3199] save

---
 scheduler/src/lib.rs | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index de1d1758433f54..64c50abb299498 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1130,6 +1130,7 @@ impl ScheduleStage {
                     if !empty_from_exec {
                         let mut processed_execution_environment = from_exec.recv().unwrap();
                         from_exec_len = from_exec_len.checked_sub(1).unwrap();
+                        empty_from_exec = from_exec_len == 0;
                         executing_queue_count = executing_queue_count.checked_sub(1).unwrap();
                         Self::commit_completed_execution(&mut processed_execution_environment, address_book, &mut execute_clock, &mut provisioning_tracker_count);
                         to_next_stage.send(processed_execution_environment).unwrap();
@@ -1137,6 +1138,7 @@ impl ScheduleStage {
                     if !empty_from {
                        let task = from.recv().unwrap();
                        from_len = from_len.checked_sub(1).unwrap();
+                       empty_from = from_len == 0;
                        Self::register_runnable_task(task, runnable_queue, &mut sequence_time);
                     }
                 }

From 571fafd5422eac0c89cfd3f37e61f9003294d70b Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 28 Aug 2022 22:10:12 +0900
Subject: [PATCH 1559/3199] save

---
 scheduler/src/lib.rs | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 64c50abb299498..4fd21757b97417 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1065,12 +1065,14 @@ impl ScheduleStage {
             }).unwrap();
         }
 
+        let (mut from_disconnected, mut from_exec_disconnected) = (false, false);
         loop {
             crossbeam_channel::select! {
                recv(from_exec) -> maybe_from_exec => {
                    if maybe_from_exec.is_err() {
                        assert_eq!(from_exec.len(), 0);
-                       if from.len() == 0 {
+                       from_exec_disconnected = true;
+                       if from_disconnected {
                            break;
                        } else {
                            continue;
@@ -1084,7 +1086,8 @@ impl ScheduleStage {
                recv(from) -> maybe_from => {
                    if maybe_from.is_err() {
                        assert_eq!(from.len(), 0);
-                       if from_exec.len() == 0 {
+                       from_disconnected = true;
+                       if from_exec_disconnected {
                            break;
                        } else {
                            continue;

From c06f986f1b4f9665de60f8107eb76627170af191 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 28 Aug 2022 22:12:41 +0900
Subject: [PATCH 1560/3199] save

---
 scheduler/src/lib.rs                    | 10 +++++-----
 transaction-scheduler-bench/src/main.rs |  2 +-
 2 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 4fd21757b97417..99c2189ec13392 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -794,10 +794,10 @@ impl ScheduleStage {
                     trace!("move to contended due to lock failure [{}/{}/{}]", unlockable_count, provisional_count, lock_count);
                     next_task.mark_as_contended();
                     *contended_count = contended_count.checked_add(1).unwrap();
-                    for lock_attempt in next_task.tx.1.iter() {
-                        lock_attempt.contended_unique_weights().insert_task(unique_weight, TaskInQueue::clone(&a2));
-                    }
-                    //task_sender.send((TaskInQueue::clone(&a2), std::mem::take(&mut next_task.for_indexer))).unwrap();
+                    //for lock_attempt in next_task.tx.1.iter() {
+                    //    lock_attempt.contended_unique_weights().insert_task(unique_weight, TaskInQueue::clone(&a2));
+                    //}
+                    task_sender.send((TaskInQueue::clone(&a2), std::mem::take(&mut next_task.for_indexer))).unwrap();
 
                     // maybe run lightweight prune logic on contended_queue here.
                 } else {
@@ -965,7 +965,7 @@ impl ScheduleStage {
     fn commit_completed_execution(ee: &mut ExecutionEnvironment, address_book: &mut AddressBook, commit_time: &mut usize, provisioning_tracker_count: &mut usize) {
         // do par()-ly?
 
-        ee.reindex();
+        //ee.reindex();
         ee.task.record_commit_time(*commit_time);
         ee.task.trace_timestamps("commit");
         //*commit_time = commit_time.checked_add(1).unwrap();
diff --git a/transaction-scheduler-bench/src/main.rs b/transaction-scheduler-bench/src/main.rs
index 6658c5010c07b8..61f814f7e0dff0 100644
--- a/transaction-scheduler-bench/src/main.rs
+++ b/transaction-scheduler-bench/src/main.rs
@@ -360,7 +360,7 @@ fn handle_transaction_batch(
         .priority_collected
         .fetch_add(priority_collected, Ordering::Relaxed);
 
-    //transaction_batch.reindex();
+    transaction_batch.reindex();
     completed_transaction_sender
         .send(transaction_batch)
         .unwrap();

From 4f7f929706e0eb2e0ea86d9188730b07c3b1717c Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 28 Aug 2022 22:22:20 +0900
Subject: [PATCH 1561/3199] save

---
 scheduler/src/lib.rs | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 99c2189ec13392..56a663225ab5f1 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -890,9 +890,9 @@ impl ScheduleStage {
                 let mut inserted = false;
 
                 if let Some(task) = l.heaviest_uncontended.take() {
-                    assert!(!task.already_finished());
-                    if true || task.currently_contended() {
-                        assert!(task.currently_contended());
+                    //assert!(!task.already_finished());
+                    if /*true ||*/ task.currently_contended() {
+                        //assert!(task.currently_contended());
                         inserted = true;
                         address_book.uncontended_task_ids.insert(task.unique_weight, task);
                     } /*else {

From 037be76b775b31b3119a9fe24213992a223d3381 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 28 Aug 2022 22:35:58 +0900
Subject: [PATCH 1562/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 56a663225ab5f1..0f18f3af419656 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -967,7 +967,7 @@ impl ScheduleStage {
 
         //ee.reindex();
         ee.task.record_commit_time(*commit_time);
-        ee.task.trace_timestamps("commit");
+        //ee.task.trace_timestamps("commit");
         //*commit_time = commit_time.checked_add(1).unwrap();
 
         // which order for data race free?: unlocking / marking

From 41255ce2582458e746c7636ee194d33530740d86 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 28 Aug 2022 22:36:43 +0900
Subject: [PATCH 1563/3199] save

---
 scheduler/src/lib.rs | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 0f18f3af419656..4039ee72a26282 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -489,7 +489,7 @@ impl Task {
     }
 
     pub fn record_sequence_time(&self, clock: usize) {
-        self.sequence_time.store(clock, std::sync::atomic::Ordering::SeqCst);
+        //self.sequence_time.store(clock, std::sync::atomic::Ordering::SeqCst);
     }
 
     pub fn sequence_time(&self) -> usize {
@@ -501,8 +501,8 @@ impl Task {
     }
 
     pub fn record_queue_time(&self, seq_clock: usize, queue_clock: usize) {
-        self.sequence_end_time.store(seq_clock, std::sync::atomic::Ordering::SeqCst);
-        self.queue_time.store(queue_clock, std::sync::atomic::Ordering::SeqCst);
+        //self.sequence_end_time.store(seq_clock, std::sync::atomic::Ordering::SeqCst);
+        //self.queue_time.store(queue_clock, std::sync::atomic::Ordering::SeqCst);
     }
 
     pub fn queue_time(&self) -> usize {
@@ -514,8 +514,8 @@ impl Task {
     }
 
     pub fn record_execute_time(&self, queue_clock: usize, execute_clock: usize) {
-        self.queue_end_time.store(queue_clock, std::sync::atomic::Ordering::SeqCst);
-        self.execute_time.store(execute_clock, std::sync::atomic::Ordering::SeqCst);
+        //self.queue_end_time.store(queue_clock, std::sync::atomic::Ordering::SeqCst);
+        //self.execute_time.store(execute_clock, std::sync::atomic::Ordering::SeqCst);
     }
 
     pub fn execute_time(&self) -> usize {
@@ -523,7 +523,7 @@ impl Task {
     }
 
     pub fn record_commit_time(&self, execute_clock: usize) {
-        self.commit_time.store(execute_clock, std::sync::atomic::Ordering::SeqCst);
+        //self.commit_time.store(execute_clock, std::sync::atomic::Ordering::SeqCst);
     }
 
     pub fn commit_time(&self) -> usize {

From 4875f6ce5e8568b93c0ebcaf09701666fb19bc47 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 28 Aug 2022 22:37:17 +0900
Subject: [PATCH 1564/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 4039ee72a26282..2753c218d50035 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1035,7 +1035,7 @@ impl ScheduleStage {
             let h = std::thread::Builder::new().name("sol-reaper".to_string()).spawn(move || {
                 while let Ok(mut a) = ee_receiver.recv() {
                     assert!(a.task.tx.1.is_empty());
-                    assert!(a.task.sequence_time() != usize::max_value());
+                    //assert!(a.task.sequence_time() != usize::max_value());
                     //let lock_attempts = std::mem::take(&mut a.lock_attempts);
                     //drop(lock_attempts);
                     //TaskInQueue::get_mut(&mut a.task).unwrap();

From d4c3ad81342845d7edde46f4d8b7069721499d60 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 29 Aug 2022 14:02:49 +0900
Subject: [PATCH 1565/3199] save

---
 scheduler/src/lib.rs | 1 +
 1 file changed, 1 insertion(+)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 2753c218d50035..57a24d6104b616 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -422,6 +422,7 @@ pub struct Preloader {
 
 impl Preloader {
     pub fn load(&self, address: Pubkey) -> PageRc {
+        self.book.get_or_insert_with();
         match self.book.entry(address) {
             AddressMapEntry::Vacant(book_entry) => {
                 let page = PageRc(MyRcInner::new(Page::new(Usage::unused())));

From 6f2e785926c06146332b7e33c9f8994ddb814603 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 29 Aug 2022 14:06:02 +0900
Subject: [PATCH 1566/3199] save

---
 scheduler/src/lib.rs | 1 -
 1 file changed, 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 57a24d6104b616..2753c218d50035 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -422,7 +422,6 @@ pub struct Preloader {
 
 impl Preloader {
     pub fn load(&self, address: Pubkey) -> PageRc {
-        self.book.get_or_insert_with();
         match self.book.entry(address) {
             AddressMapEntry::Vacant(book_entry) => {
                 let page = PageRc(MyRcInner::new(Page::new(Usage::unused())));

From e4c417f0daa5521d5a4411785f862f9a3b496aa7 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 29 Aug 2022 14:23:07 +0900
Subject: [PATCH 1567/3199] save

---
 scheduler/src/lib.rs | 13 +------------
 1 file changed, 1 insertion(+), 12 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 2753c218d50035..a4d48ea9143f9e 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -422,18 +422,7 @@ pub struct Preloader {
 
 impl Preloader {
     pub fn load(&self, address: Pubkey) -> PageRc {
-        match self.book.entry(address) {
-            AddressMapEntry::Vacant(book_entry) => {
-                let page = PageRc(MyRcInner::new(Page::new(Usage::unused())));
-                let cloned = PageRc::clone(&page);
-                book_entry.insert(page);
-                cloned
-            }
-            AddressMapEntry::Occupied(mut book_entry) => {
-                let page = book_entry.get();
-                PageRc::clone(&page)
-            }
-        }
+        self.book.entry(address).or_insert_with(|| PageRc(MyRcInner::new(Page::new(Usage::unused()))))
     }
 }
 

From 3afaebd6366cb74ae15ece8cf4c78475395136a7 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 29 Aug 2022 14:23:22 +0900
Subject: [PATCH 1568/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index a4d48ea9143f9e..0507ae657903a7 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -422,7 +422,7 @@ pub struct Preloader {
 
 impl Preloader {
     pub fn load(&self, address: Pubkey) -> PageRc {
-        self.book.entry(address).or_insert_with(|| PageRc(MyRcInner::new(Page::new(Usage::unused()))))
+        PageRc::clone(self.book.entry(address).or_insert_with(|| PageRc(MyRcInner::new(Page::new(Usage::unused())))))
     }
 }
 

From 903402646d8350250568ae6972d55123aeb64bc7 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 29 Aug 2022 14:23:30 +0900
Subject: [PATCH 1569/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 0507ae657903a7..9dc389c02f46be 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -422,7 +422,7 @@ pub struct Preloader {
 
 impl Preloader {
     pub fn load(&self, address: Pubkey) -> PageRc {
-        PageRc::clone(self.book.entry(address).or_insert_with(|| PageRc(MyRcInner::new(Page::new(Usage::unused())))))
+        PageRc::clone(&self.book.entry(address).or_insert_with(|| PageRc(MyRcInner::new(Page::new(Usage::unused())))))
     }
 }
 

From ac0f6cbac2906b2b610a8c61bc72fcdb92aea48e Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 29 Aug 2022 14:32:45 +0900
Subject: [PATCH 1570/3199] save

---
 scheduler/src/lib.rs | 1 +
 1 file changed, 1 insertion(+)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 9dc389c02f46be..9521d3ff7a4134 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -421,6 +421,7 @@ pub struct Preloader {
 }
 
 impl Preloader {
+    #[inline(never)]
     pub fn load(&self, address: Pubkey) -> PageRc {
         PageRc::clone(&self.book.entry(address).or_insert_with(|| PageRc(MyRcInner::new(Page::new(Usage::unused())))))
     }

From 954d71350292a31aa3a681037d5bc0830f4a521f Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 2 Sep 2022 21:40:13 +0900
Subject: [PATCH 1571/3199] save

---
 scheduler/src/lib.rs | 43 ++++++++++++++++++++++++++++++++-----------
 1 file changed, 32 insertions(+), 11 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 9521d3ff7a4134..6a2ce0800262be 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -21,10 +21,12 @@ type MyRcInner<T> = std::rc::Rc<T>;
 unsafe impl Send for PageRc {}
 */
 
-type MyRcInner = std::sync::Arc<Page>;
+type MyRcInner = std::sync::Arc<std::cell::RefCell<Page>>;
 
 #[derive(Debug, Clone)]
 pub struct PageRc(MyRcInner);
+unsafe impl Send for PageRc {}
+unsafe impl Sync for PageRc {}
 
 
 #[derive(Debug)]
@@ -93,13 +95,15 @@ impl ExecutionEnvironment {
 }
 
 impl PageRc {
-    fn page_mut(&mut self) -> &mut Page {
-        unsafe { MyRcInner::get_mut_unchecked(&mut self.0) }
+    fn page_mut(&mut self) -> std::cell::RefMut<'_, Page> {
+        //unsafe { MyRcInner::get_mut_unchecked(&mut self.0) }
+        self.0.borrow_mut()
     }
 
-    fn page_ref(&self) -> &Page {
+    fn page_ref(&self) -> std::cell::Ref<'_, Page> {
         //<MyRcInner as std::borrow::Borrow<_>>::borrow(&self.0)
-        &*(self.0)
+        self.0.borrow()
+        //&*(self.0)
     }
 }
 
@@ -142,7 +146,8 @@ impl LockAttempt {
     }
 
     pub fn contended_unique_weights(&self) -> &TaskIds {
-        &self.target.page_ref().contended_unique_weights
+        //&self.target.page_ref().contended_unique_weights
+        panic!()
     }
 }
 
@@ -209,7 +214,7 @@ impl TaskIds {
 pub struct Page {
     current_usage: Usage,
     next_usage: Usage,
-    pub contended_unique_weights: TaskIds,
+    contended_unique_weights: TaskIds,
     provisional_task_ids: Vec<std::sync::Arc<ProvisioningTracker>>,
     //loaded account from Accounts db
     //comulative_cu for qos; i.e. track serialized cumulative keyed by addresses and bail out block
@@ -287,6 +292,7 @@ impl AddressBook {
 
                 let mut page = target.page_mut();
 
+                let next_usage = page.next_usage;
                 match page.current_usage {
                     Usage::Unused => {
                         assert_eq!(page.next_usage, Usage::Unused);
@@ -296,7 +302,7 @@ impl AddressBook {
                     Usage::Readonly(ref mut count) => match requested_usage {
                         RequestedUsage::Readonly => {
                             // prevent newer read-locks (even from runnable too)
-                            match page.next_usage {
+                            match next_usage {
                                 Usage::Unused => {
                                     *count += 1;
                                     *status = LockStatus::Succeded;
@@ -423,7 +429,7 @@ pub struct Preloader {
 impl Preloader {
     #[inline(never)]
     pub fn load(&self, address: Pubkey) -> PageRc {
-        PageRc::clone(&self.book.entry(address).or_insert_with(|| PageRc(MyRcInner::new(Page::new(Usage::unused())))))
+        PageRc::clone(&self.book.entry(address).or_insert_with(|| PageRc(MyRcInner::new(core::cell::RefCell::new(Page::new(Usage::unused()))))))
     }
 }
 
@@ -475,7 +481,19 @@ pub struct Task {
 
 impl Task {
     pub fn new_for_queue(unique_weight: UniqueWeight, tx: (SanitizedTransaction, Vec<LockAttempt>)) -> std::sync::Arc<Self> {
-        TaskInQueue::new(Self { for_indexer: tx.1.iter().map(|a| a.clone_for_test()).collect(), unique_weight, tx, contention_count: 0, uncontended: Default::default(), sequence_time: std::sync::atomic::AtomicUsize::new(usize::max_value()), sequence_end_time: std::sync::atomic::AtomicUsize::new(usize::max_value()), queue_time: std::sync::atomic::AtomicUsize::new(usize::max_value()), queue_end_time: std::sync::atomic::AtomicUsize::new(usize::max_value()), execute_time: std::sync::atomic::AtomicUsize::new(usize::max_value()), commit_time: std::sync::atomic::AtomicUsize::new(usize::max_value()),  })
+        TaskInQueue::new(Self {
+            for_indexer: tx.1.iter().map(|a| a.clone_for_test()).collect(),
+            unique_weight,
+            tx,
+            contention_count: 0,
+            uncontended: Default::default(),
+            sequence_time: std::sync::atomic::AtomicUsize::new(usize::max_value()),
+            sequence_end_time: std::sync::atomic::AtomicUsize::new(usize::max_value()),
+            queue_time: std::sync::atomic::AtomicUsize::new(usize::max_value()),
+            queue_end_time: std::sync::atomic::AtomicUsize::new(usize::max_value()),
+            execute_time: std::sync::atomic::AtomicUsize::new(usize::max_value()),
+            commit_time: std::sync::atomic::AtomicUsize::new(usize::max_value())
+        })
     }
 
     pub fn record_sequence_time(&self, clock: usize) {
@@ -875,7 +893,7 @@ impl ScheduleStage {
         for mut l in lock_attempts.into_iter() {
             let newly_uncontended = address_book.reset_lock(&mut l, true);
 
-            let page = l.target.page_mut();
+            let mut page = l.target.page_mut();
             if newly_uncontended && page.next_usage == Usage::Unused {
                 let mut inserted = false;
 
@@ -913,6 +931,8 @@ impl ScheduleStage {
             if page.current_usage == Usage::Unused && page.next_usage != Usage::Unused {
                 page.switch_to_next_usage();
                 for mut tracker in std::mem::take(&mut page.provisional_task_ids).into_iter() {
+                    panic!();
+                    /*
                     let tracker = unsafe { std::sync::Arc::<ProvisioningTracker>::get_mut_unchecked(&mut tracker) };
                     tracker.progress();
                     if tracker.is_fulfilled() {
@@ -922,6 +942,7 @@ impl ScheduleStage {
                     } else {
                         trace!("provisioning tracker progress: {} => {}", tracker.prev_count(), tracker.count());
                     }
+                    */
                 }
             }
 

From 06cc62cacce8f4a588e956683582cebb122bc501 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 2 Sep 2022 21:44:02 +0900
Subject: [PATCH 1572/3199] save

---
 scheduler/src/lib.rs | 1 -
 1 file changed, 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 6a2ce0800262be..5985d2f35b6f26 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1,5 +1,4 @@
 #![feature(map_first_last)]
-#![feature(get_mut_unchecked)]
 
 use {
     crossbeam_channel::{bounded, unbounded},

From eca560f61168be344b1a67bd5458d68eea9e323d Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 2 Sep 2022 22:12:00 +0900
Subject: [PATCH 1573/3199] save

---
 scheduler/src/lib.rs | 17 ++++++++++++++---
 1 file changed, 14 insertions(+), 3 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 5985d2f35b6f26..f29549ea8581cf 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1,4 +1,6 @@
 #![feature(map_first_last)]
+#![feature(negative_impls)]
+//#![feature(get_mut_unchecked)]
 
 use {
     crossbeam_channel::{bounded, unbounded},
@@ -93,8 +95,10 @@ impl ExecutionEnvironment {
     }
 }
 
+unsafe trait AtScheduleThread {}
+
 impl PageRc {
-    fn page_mut(&mut self) -> std::cell::RefMut<'_, Page> {
+    fn page_mut/*<ST: AtScheduleThread>*/(&mut self) -> std::cell::RefMut<'_, Page> {
         //unsafe { MyRcInner::get_mut_unchecked(&mut self.0) }
         self.0.borrow_mut()
     }
@@ -676,6 +680,12 @@ pub fn get_transaction_priority_details(tx: &SanitizedTransaction) -> u64 {
             .map(|d| d.get_priority()).unwrap_or_default()
 }
 
+#[derive(Debug)]
+struct AtTopOfScheduleThread;
+unsafe impl AtScheduleThread for AtTopOfScheduleThread {}
+impl !Send for AtTopOfScheduleThread {}
+impl !Sync for AtTopOfScheduleThread {}
+
 pub struct ScheduleStage {}
 
 impl ScheduleStage {
@@ -754,7 +764,7 @@ impl ScheduleStage {
     ) -> Option<(UniqueWeight, TaskInQueue, Vec<LockAttempt>)> {
         if let Some(mut a) = address_book.fulfilled_provisional_task_ids.pop_last() {
             trace!("expediate pop from provisional queue [rest: {}]", address_book.fulfilled_provisional_task_ids.len());
-            let next_task = unsafe { TaskInQueue::get_mut_unchecked(&mut a.1) };
+            let next_task: &mut TaskInQueue = unsafe { panic!(); /*TaskInQueue::get_mut_unchecked(&mut a.1)*/ };
 
             let lock_attempts = std::mem::take(&mut next_task.tx.1);
 
@@ -770,7 +780,7 @@ impl ScheduleStage {
                 *queue_clock = queue_clock.checked_add(1).unwrap();
             }
             let a2 = TaskInQueue::clone(&arc_next_task);
-            let next_task = unsafe { TaskInQueue::get_mut_unchecked(&mut arc_next_task) };
+            let next_task: &mut TaskInQueue = unsafe { panic!(); /*TaskInQueue::get_mut_unchecked(&mut arc_next_task)*/ };
             let unique_weight = next_task.unique_weight;
             let message_hash = next_task.tx.0.message_hash();
 
@@ -1031,6 +1041,7 @@ impl ScheduleStage {
         to_execute_substage: &crossbeam_channel::Sender<Box<ExecutionEnvironment>>,
         maybe_to_next_stage: Option<&crossbeam_channel::Sender<Box<ExecutionEnvironment>>>, // assume nonblocking
     ) {
+
         let mut executing_queue_count = 0_usize;
         let mut contended_count = 0;
         let mut provisioning_tracker_count = 0;

From d55a588e96e8ab57bd5a9e479245c11d3829e682 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 2 Sep 2022 22:13:49 +0900
Subject: [PATCH 1574/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index f29549ea8581cf..91d608a24aa906 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1032,7 +1032,7 @@ impl ScheduleStage {
         Self::push_to_runnable_queue(weighted_tx, runnable_queue)
     }
 
-    pub fn run(
+    pub fn run<AST = AtTopOfScheduleThread>(
         max_executing_queue_count: usize,
         runnable_queue: &mut TaskQueue,
         address_book: &mut AddressBook,

From cf4ae7f0cca24da557c65d99d3774c0d0b624cd9 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 2 Sep 2022 22:34:12 +0900
Subject: [PATCH 1575/3199] save

---
 scheduler/src/lib.rs | 94 ++++++++++++++++++++++++++------------------
 1 file changed, 56 insertions(+), 38 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 91d608a24aa906..d491807828f23d 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -98,7 +98,7 @@ impl ExecutionEnvironment {
 unsafe trait AtScheduleThread {}
 
 impl PageRc {
-    fn page_mut/*<ST: AtScheduleThread>*/(&mut self) -> std::cell::RefMut<'_, Page> {
+    fn page_mut<ST: AtScheduleThread>(&mut self) -> std::cell::RefMut<'_, Page> {
         //unsafe { MyRcInner::get_mut_unchecked(&mut self.0) }
         self.0.borrow_mut()
     }
@@ -285,7 +285,7 @@ impl ProvisioningTracker {
 
 impl AddressBook {
     #[inline(never)]
-    fn attempt_lock_address(
+    fn attempt_lock_address<AST: AtScheduleThread>(
         from_runnable: bool,
         prefer_immediate: bool,
         unique_weight: &UniqueWeight,
@@ -293,7 +293,7 @@ impl AddressBook {
     ) {
         let LockAttempt {target, requested_usage, status/*, remembered*/, ..} = attempt;
 
-                let mut page = target.page_mut();
+                let mut page = target.page_mut::<AST>();
 
                 let next_usage = page.next_usage;
                 match page.current_usage {
@@ -351,16 +351,16 @@ impl AddressBook {
                 }
     }
 
-    fn reset_lock(&mut self, attempt: &mut LockAttempt, after_execution: bool) -> bool {
+    fn reset_lock<AST: AtScheduleThread>(&mut self, attempt: &mut LockAttempt, after_execution: bool) -> bool {
         match attempt.status {
             LockStatus::Succeded => {
-                self.unlock(attempt)
+                self.unlock::<AST>(attempt)
             },
             LockStatus::Provisional => {
                 if after_execution {
-                    self.unlock(attempt)
+                    self.unlock::<AST>(attempt)
                 } else {
-                    self.cancel(attempt);
+                    self.cancel::<AST>(attempt);
                     false
                 }
             }
@@ -371,12 +371,12 @@ impl AddressBook {
     }
 
     #[inline(never)]
-    fn unlock(&mut self, attempt: &mut LockAttempt) -> bool {
+    fn unlock<AST: AtScheduleThread>(&mut self, attempt: &mut LockAttempt) -> bool {
         //debug_assert!(attempt.is_success());
 
         let mut newly_uncontended = false;
 
-        let mut page = attempt.target.page_mut();
+        let mut page = attempt.target.page_mut::<AST>();
 
         match &mut page.current_usage {
             Usage::Readonly(ref mut count) => match &attempt.requested_usage {
@@ -406,8 +406,8 @@ impl AddressBook {
     }
 
     #[inline(never)]
-    fn cancel(&mut self, attempt: &mut LockAttempt) {
-        let mut page = attempt.target.page_mut();
+    fn cancel<AST: AtScheduleThread>(&mut self, attempt: &mut LockAttempt) {
+        let mut page = attempt.target.page_mut::<AST>();
 
         match page.next_usage {
             Usage::Unused => {
@@ -638,7 +638,7 @@ impl TaskQueue {
 }
 
 #[inline(never)]
-fn attempt_lock_for_execution<'a>(
+fn attempt_lock_for_execution<'a, AST: AtScheduleThread>(
     from_runnable: bool,
     prefer_immediate: bool,
     address_book: &mut AddressBook,
@@ -651,7 +651,7 @@ fn attempt_lock_for_execution<'a>(
     let mut provisional_count = 0;
 
     for attempt in placeholder_attempts.iter_mut() {
-        AddressBook::attempt_lock_address(from_runnable, prefer_immediate, unique_weight, attempt);
+        AddressBook::attempt_lock_address::<AST>(from_runnable, prefer_immediate, unique_weight, attempt);
         match attempt.status {
             LockStatus::Succeded => {},
             LockStatus::Failed => {
@@ -680,12 +680,6 @@ pub fn get_transaction_priority_details(tx: &SanitizedTransaction) -> u64 {
             .map(|d| d.get_priority()).unwrap_or_default()
 }
 
-#[derive(Debug)]
-struct AtTopOfScheduleThread;
-unsafe impl AtScheduleThread for AtTopOfScheduleThread {}
-impl !Send for AtTopOfScheduleThread {}
-impl !Sync for AtTopOfScheduleThread {}
-
 pub struct ScheduleStage {}
 
 impl ScheduleStage {
@@ -752,7 +746,7 @@ impl ScheduleStage {
     }
 
     #[inline(never)]
-    fn pop_from_queue_then_lock(
+    fn pop_from_queue_then_lock<AST: AtScheduleThread>(
         task_sender: &crossbeam_channel::Sender<(TaskInQueue, Vec<LockAttempt>)>,
         runnable_queue: &mut TaskQueue,
         address_book: &mut AddressBook,
@@ -787,7 +781,7 @@ impl ScheduleStage {
             // plumb message_hash into StatusCache or implmenent our own for duplicate tx
             // detection?
 
-            let (unlockable_count, provisional_count) = attempt_lock_for_execution(
+            let (unlockable_count, provisional_count) = attempt_lock_for_execution::<AST>(
                 from_runnable,
                 prefer_immediate,
                 address_book,
@@ -798,7 +792,7 @@ impl ScheduleStage {
 
             if unlockable_count > 0 {
                 //trace!("reset_lock_for_failed_execution(): {:?} {}", (&unique_weight, from_runnable), next_task.tx.0.signature());
-                Self::reset_lock_for_failed_execution(
+                Self::reset_lock_for_failed_execution::<AST>(
                     address_book,
                     &unique_weight,
                     &mut next_task.tx.1,
@@ -836,7 +830,7 @@ impl ScheduleStage {
                 next_task.mark_as_uncontended();
                 let tracker = std::sync::Arc::new(ProvisioningTracker::new(provisional_count, TaskInQueue::clone(&a2)));
                 *provisioning_tracker_count = provisioning_tracker_count.checked_add(1).unwrap();
-                Self::finalize_lock_for_provisional_execution(
+                Self::finalize_lock_for_provisional_execution::<AST>(
                     address_book,
                     next_task,
                     tracker
@@ -864,7 +858,7 @@ impl ScheduleStage {
     }
 
     #[inline(never)]
-    fn finalize_lock_for_provisional_execution(
+    fn finalize_lock_for_provisional_execution<AST: AtScheduleThread>(
         address_book: &mut AddressBook,
         next_task: &mut Task,
         tracker: std::sync::Arc<ProvisioningTracker>,
@@ -872,7 +866,7 @@ impl ScheduleStage {
         for l in next_task.tx.1.iter_mut() {
             match l.status {
                 LockStatus::Provisional => {
-                    l.target.page_mut().provisional_task_ids.push(std::sync::Arc::clone(&tracker));
+                    l.target.page_mut::<AST>().provisional_task_ids.push(std::sync::Arc::clone(&tracker));
                 }
                 LockStatus::Succeded => {
                     // do nothing
@@ -886,23 +880,23 @@ impl ScheduleStage {
     }
 
     #[inline(never)]
-    fn reset_lock_for_failed_execution(
+    fn reset_lock_for_failed_execution<AST: AtScheduleThread>(
         address_book: &mut AddressBook,
         unique_weight: &UniqueWeight,
         lock_attempts: &mut Vec<LockAttempt>,
         from_runnable: bool,
     ) {
         for l in lock_attempts {
-            address_book.reset_lock(l, false);
+            address_book.reset_lock::<AST>(l, false);
         }
     }
 
     #[inline(never)]
-    fn unlock_after_execution(address_book: &mut AddressBook, lock_attempts: &mut Vec<LockAttempt>, provisioning_tracker_count: &mut usize) {
+    fn unlock_after_execution<AST: AtScheduleThread>(address_book: &mut AddressBook, lock_attempts: &mut Vec<LockAttempt>, provisioning_tracker_count: &mut usize) {
         for mut l in lock_attempts.into_iter() {
-            let newly_uncontended = address_book.reset_lock(&mut l, true);
+            let newly_uncontended = address_book.reset_lock::<AST>(&mut l, true);
 
-            let mut page = l.target.page_mut();
+            let mut page = l.target.page_mut::<AST>();
             if newly_uncontended && page.next_usage == Usage::Unused {
                 let mut inserted = false;
 
@@ -981,8 +975,12 @@ impl ScheduleStage {
         })
     }
 
+    fn commit_completed_execution2<AST: AtScheduleThread>() {
+        panic!();
+    }
+
     #[inline(never)]
-    fn commit_completed_execution(ee: &mut ExecutionEnvironment, address_book: &mut AddressBook, commit_time: &mut usize, provisioning_tracker_count: &mut usize) {
+    fn commit_completed_execution<AST: AtScheduleThread>(ee: &mut ExecutionEnvironment, address_book: &mut AddressBook, commit_time: &mut usize, provisioning_tracker_count: &mut usize) {
         // do par()-ly?
 
         //ee.reindex();
@@ -991,7 +989,7 @@ impl ScheduleStage {
         //*commit_time = commit_time.checked_add(1).unwrap();
 
         // which order for data race free?: unlocking / marking
-        Self::unlock_after_execution(address_book, &mut ee.lock_attempts, provisioning_tracker_count);
+        Self::unlock_after_execution::<AST>(address_book, &mut ee.lock_attempts, provisioning_tracker_count);
         ee.task.mark_as_finished();
 
         // block-wide qos validation will be done here
@@ -1004,7 +1002,7 @@ impl ScheduleStage {
     }
 
     #[inline(never)]
-    fn schedule_next_execution(
+    fn schedule_next_execution<AST: AtScheduleThread>(
         task_sender: &crossbeam_channel::Sender<(TaskInQueue, Vec<LockAttempt>)>,
         runnable_queue: &mut TaskQueue,
         address_book: &mut AddressBook,
@@ -1016,7 +1014,7 @@ impl ScheduleStage {
         provisioning_tracker_count: &mut usize,
     ) -> Option<Box<ExecutionEnvironment>> {
         let maybe_ee =
-            Self::pop_from_queue_then_lock(task_sender, runnable_queue, address_book, contended_count, prefer_immediate, sequence_time, queue_clock, provisioning_tracker_count)
+            Self::pop_from_queue_then_lock::<AST>(task_sender, runnable_queue, address_book, contended_count, prefer_immediate, sequence_time, queue_clock, provisioning_tracker_count)
                 .map(|(uw, t,ll)| Self::prepare_scheduled_execution(address_book, uw, t, ll, queue_clock, execute_clock));
         maybe_ee
     }
@@ -1032,7 +1030,8 @@ impl ScheduleStage {
         Self::push_to_runnable_queue(weighted_tx, runnable_queue)
     }
 
-    pub fn run<AST = AtTopOfScheduleThread>(
+    fn _run<AST: AtScheduleThread>(
+        ast: AST,
         max_executing_queue_count: usize,
         runnable_queue: &mut TaskQueue,
         address_book: &mut AddressBook,
@@ -1082,6 +1081,7 @@ impl ScheduleStage {
                         lock_attempt.contended_unique_weights().insert_task(task.unique_weight, TaskInQueue::clone(&task));
                     }
                 }
+                Self::commit_completed_execution2::<AST>(ast);
                 assert_eq!(task_receiver.len(), 0);
             }).unwrap();
         }
@@ -1101,7 +1101,7 @@ impl ScheduleStage {
                    }
                    let mut processed_execution_environment = maybe_from_exec.unwrap();
                     executing_queue_count = executing_queue_count.checked_sub(1).unwrap();
-                    Self::commit_completed_execution(&mut processed_execution_environment, address_book, &mut execute_clock, &mut provisioning_tracker_count);
+                    Self::commit_completed_execution::<AST>(&mut processed_execution_environment, address_book, &mut execute_clock, &mut provisioning_tracker_count);
                     to_next_stage.send(processed_execution_environment).unwrap();
                }
                recv(from) -> maybe_from => {
@@ -1127,7 +1127,7 @@ impl ScheduleStage {
                 while (executing_queue_count + provisioning_tracker_count) < max_executing_queue_count {
                     trace!("schedule_once (from: {}, to: {}, runnnable: {}, contended: {}, (immediate+provisional)/max: ({}+{})/{}) active from contended: {}!", from.len(), to_execute_substage.len(), runnable_queue.task_count(), contended_count, executing_queue_count, provisioning_tracker_count, max_executing_queue_count, address_book.uncontended_task_ids.len());
                     let prefer_immediate = provisioning_tracker_count/4 > executing_queue_count;
-                    if let Some(ee) = Self::schedule_next_execution(&task_sender, runnable_queue, address_book, &mut contended_count, prefer_immediate, &sequence_time, &mut queue_clock, &mut execute_clock, &mut provisioning_tracker_count) {
+                    if let Some(ee) = Self::schedule_next_execution::<AST>(&task_sender, runnable_queue, address_book, &mut contended_count, prefer_immediate, &sequence_time, &mut queue_clock, &mut execute_clock, &mut provisioning_tracker_count) {
                         executing_queue_count = executing_queue_count.checked_add(1).unwrap();
                         to_execute_substage.send(ee).unwrap();
                     } else {
@@ -1156,7 +1156,7 @@ impl ScheduleStage {
                         from_exec_len = from_exec_len.checked_sub(1).unwrap();
                         empty_from_exec = from_exec_len == 0;
                         executing_queue_count = executing_queue_count.checked_sub(1).unwrap();
-                        Self::commit_completed_execution(&mut processed_execution_environment, address_book, &mut execute_clock, &mut provisioning_tracker_count);
+                        Self::commit_completed_execution::<AST>(&mut processed_execution_environment, address_book, &mut execute_clock, &mut provisioning_tracker_count);
                         to_next_stage.send(processed_execution_environment).unwrap();
                     }
                     if !empty_from {
@@ -1170,6 +1170,24 @@ impl ScheduleStage {
         }
         info!("run finished...");
     }
+
+    pub fn run(
+        max_executing_queue_count: usize,
+        runnable_queue: &mut TaskQueue,
+        address_book: &mut AddressBook,
+        from: &crossbeam_channel::Receiver<TaskInQueue>,
+        from_exec: &crossbeam_channel::Receiver<Box<ExecutionEnvironment>>,
+        to_execute_substage: &crossbeam_channel::Sender<Box<ExecutionEnvironment>>,
+        maybe_to_next_stage: Option<&crossbeam_channel::Sender<Box<ExecutionEnvironment>>>, // assume nonblocking
+    ) {
+        #[derive(Debug)]
+        struct AtTopOfScheduleThread;
+        unsafe impl AtScheduleThread for AtTopOfScheduleThread {}
+        impl !Send for AtTopOfScheduleThread {}
+        impl !Sync for AtTopOfScheduleThread {}
+
+        Self::_run::<AtTopOfScheduleThread>(AtTopOfScheduleThread, max_executing_queue_count, runnable_queue, address_book, from, from_exec, to_execute_substage, maybe_to_next_stage)
+    }
 }
 
 struct ExecuteStage {

From 9080ad2b712b2498afbb2bd8f0830a12183a733e Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 2 Sep 2022 22:34:40 +0900
Subject: [PATCH 1576/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index d491807828f23d..b9fd2639f63f98 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -975,7 +975,7 @@ impl ScheduleStage {
         })
     }
 
-    fn commit_completed_execution2<AST: AtScheduleThread>() {
+    fn commit_completed_execution2<AST: AtScheduleThread>(_a: AST) {
         panic!();
     }
 

From c0961ac7fdb57aed77fef5b2e540ff2d1e0fca52 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 2 Sep 2022 22:35:24 +0900
Subject: [PATCH 1577/3199] save

---
 scheduler/src/lib.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index b9fd2639f63f98..1802e0aa88aad0 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1183,8 +1183,8 @@ impl ScheduleStage {
         #[derive(Debug)]
         struct AtTopOfScheduleThread;
         unsafe impl AtScheduleThread for AtTopOfScheduleThread {}
-        impl !Send for AtTopOfScheduleThread {}
-        impl !Sync for AtTopOfScheduleThread {}
+        //impl !Send for AtTopOfScheduleThread {}
+        //impl !Sync for AtTopOfScheduleThread {}
 
         Self::_run::<AtTopOfScheduleThread>(AtTopOfScheduleThread, max_executing_queue_count, runnable_queue, address_book, from, from_exec, to_execute_substage, maybe_to_next_stage)
     }

From 741f4eb1ff92432f8ad72fac4b94e965a619d5c0 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 2 Sep 2022 22:37:36 +0900
Subject: [PATCH 1578/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 1802e0aa88aad0..93e967acac7359 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1081,10 +1081,10 @@ impl ScheduleStage {
                         lock_attempt.contended_unique_weights().insert_task(task.unique_weight, TaskInQueue::clone(&task));
                     }
                 }
-                Self::commit_completed_execution2::<AST>(ast);
                 assert_eq!(task_receiver.len(), 0);
             }).unwrap();
         }
+                Self::commit_completed_execution2::<AST>(ast);
 
         let (mut from_disconnected, mut from_exec_disconnected) = (false, false);
         loop {

From 69fd48a6970e4f61dbce53fe4baddafaba0e9b94 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 2 Sep 2022 22:38:11 +0900
Subject: [PATCH 1579/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 93e967acac7359..a4ee7cd9bc7d13 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1084,7 +1084,7 @@ impl ScheduleStage {
                 assert_eq!(task_receiver.len(), 0);
             }).unwrap();
         }
-                Self::commit_completed_execution2::<AST>(ast);
+                Self::commit_completed_execution2(ast);
 
         let (mut from_disconnected, mut from_exec_disconnected) = (false, false);
         loop {

From fcbb944056308c9e30a4fb6d8f35f27d61e4b783 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 2 Sep 2022 22:38:35 +0900
Subject: [PATCH 1580/3199] save

---
 scheduler/src/lib.rs | 1 -
 1 file changed, 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index a4ee7cd9bc7d13..b5998ba993d0bb 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1084,7 +1084,6 @@ impl ScheduleStage {
                 assert_eq!(task_receiver.len(), 0);
             }).unwrap();
         }
-                Self::commit_completed_execution2(ast);
 
         let (mut from_disconnected, mut from_exec_disconnected) = (false, false);
         loop {

From 7f50ad1c724ce7c8039f2e840adf5254e75a56b4 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 2 Sep 2022 22:39:48 +0900
Subject: [PATCH 1581/3199] save

---
 scheduler/src/lib.rs | 15 ++++++++++-----
 1 file changed, 10 insertions(+), 5 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index b5998ba993d0bb..dd4d8cd99745ae 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -286,6 +286,7 @@ impl ProvisioningTracker {
 impl AddressBook {
     #[inline(never)]
     fn attempt_lock_address<AST: AtScheduleThread>(
+        ast: AST,
         from_runnable: bool,
         prefer_immediate: bool,
         unique_weight: &UniqueWeight,
@@ -351,7 +352,7 @@ impl AddressBook {
                 }
     }
 
-    fn reset_lock<AST: AtScheduleThread>(&mut self, attempt: &mut LockAttempt, after_execution: bool) -> bool {
+    fn reset_lock<AST: AtScheduleThread>(&mut self, ast: AST, attempt: &mut LockAttempt, after_execution: bool) -> bool {
         match attempt.status {
             LockStatus::Succeded => {
                 self.unlock::<AST>(attempt)
@@ -371,7 +372,7 @@ impl AddressBook {
     }
 
     #[inline(never)]
-    fn unlock<AST: AtScheduleThread>(&mut self, attempt: &mut LockAttempt) -> bool {
+    fn unlock<AST: AtScheduleThread>(&mut self, ast: AST, attempt: &mut LockAttempt) -> bool {
         //debug_assert!(attempt.is_success());
 
         let mut newly_uncontended = false;
@@ -406,7 +407,7 @@ impl AddressBook {
     }
 
     #[inline(never)]
-    fn cancel<AST: AtScheduleThread>(&mut self, attempt: &mut LockAttempt) {
+    fn cancel<AST: AtScheduleThread>(&mut self, ast: AST, attempt: &mut LockAttempt) {
         let mut page = attempt.target.page_mut::<AST>();
 
         match page.next_usage {
@@ -639,6 +640,7 @@ impl TaskQueue {
 
 #[inline(never)]
 fn attempt_lock_for_execution<'a, AST: AtScheduleThread>(
+    ast: AST,
     from_runnable: bool,
     prefer_immediate: bool,
     address_book: &mut AddressBook,
@@ -747,6 +749,7 @@ impl ScheduleStage {
 
     #[inline(never)]
     fn pop_from_queue_then_lock<AST: AtScheduleThread>(
+        ast: AST,
         task_sender: &crossbeam_channel::Sender<(TaskInQueue, Vec<LockAttempt>)>,
         runnable_queue: &mut TaskQueue,
         address_book: &mut AddressBook,
@@ -859,6 +862,7 @@ impl ScheduleStage {
 
     #[inline(never)]
     fn finalize_lock_for_provisional_execution<AST: AtScheduleThread>(
+        ast: AST,
         address_book: &mut AddressBook,
         next_task: &mut Task,
         tracker: std::sync::Arc<ProvisioningTracker>,
@@ -881,6 +885,7 @@ impl ScheduleStage {
 
     #[inline(never)]
     fn reset_lock_for_failed_execution<AST: AtScheduleThread>(
+        ast: AST,
         address_book: &mut AddressBook,
         unique_weight: &UniqueWeight,
         lock_attempts: &mut Vec<LockAttempt>,
@@ -892,7 +897,7 @@ impl ScheduleStage {
     }
 
     #[inline(never)]
-    fn unlock_after_execution<AST: AtScheduleThread>(address_book: &mut AddressBook, lock_attempts: &mut Vec<LockAttempt>, provisioning_tracker_count: &mut usize) {
+    fn unlock_after_execution<AST: AtScheduleThread>(ast: AST, address_book: &mut AddressBook, lock_attempts: &mut Vec<LockAttempt>, provisioning_tracker_count: &mut usize) {
         for mut l in lock_attempts.into_iter() {
             let newly_uncontended = address_book.reset_lock::<AST>(&mut l, true);
 
@@ -980,7 +985,7 @@ impl ScheduleStage {
     }
 
     #[inline(never)]
-    fn commit_completed_execution<AST: AtScheduleThread>(ee: &mut ExecutionEnvironment, address_book: &mut AddressBook, commit_time: &mut usize, provisioning_tracker_count: &mut usize) {
+    fn commit_completed_execution<AST: AtScheduleThread>(ast: AST, ee: &mut ExecutionEnvironment, address_book: &mut AddressBook, commit_time: &mut usize, provisioning_tracker_count: &mut usize) {
         // do par()-ly?
 
         //ee.reindex();

From a80dec42c904c0f398d283b14986f561b341a77e Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 2 Sep 2022 22:40:00 +0900
Subject: [PATCH 1582/3199] save

---
 scheduler/src/lib.rs | 1 +
 1 file changed, 1 insertion(+)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index dd4d8cd99745ae..5f2b95ec693557 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1008,6 +1008,7 @@ impl ScheduleStage {
 
     #[inline(never)]
     fn schedule_next_execution<AST: AtScheduleThread>(
+        ast: AST,
         task_sender: &crossbeam_channel::Sender<(TaskInQueue, Vec<LockAttempt>)>,
         runnable_queue: &mut TaskQueue,
         address_book: &mut AddressBook,

From c6ebee081d78450883be97e750e912bf15006420 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 2 Sep 2022 22:41:51 +0900
Subject: [PATCH 1583/3199] save

---
 scheduler/src/lib.rs | 41 ++++++++++++++++++++++-------------------
 1 file changed, 22 insertions(+), 19 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 5f2b95ec693557..b636b6ba7e7068 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -294,7 +294,7 @@ impl AddressBook {
     ) {
         let LockAttempt {target, requested_usage, status/*, remembered*/, ..} = attempt;
 
-                let mut page = target.page_mut::<AST>();
+                let mut page = target.page_mut(ast);
 
                 let next_usage = page.next_usage;
                 match page.current_usage {
@@ -355,13 +355,13 @@ impl AddressBook {
     fn reset_lock<AST: AtScheduleThread>(&mut self, ast: AST, attempt: &mut LockAttempt, after_execution: bool) -> bool {
         match attempt.status {
             LockStatus::Succeded => {
-                self.unlock::<AST>(attempt)
+                self.unlock(ast, attempt)
             },
             LockStatus::Provisional => {
                 if after_execution {
-                    self.unlock::<AST>(attempt)
+                    self.unlock(ast, attempt)
                 } else {
-                    self.cancel::<AST>(attempt);
+                    self.cancel(ast, attempt);
                     false
                 }
             }
@@ -377,7 +377,7 @@ impl AddressBook {
 
         let mut newly_uncontended = false;
 
-        let mut page = attempt.target.page_mut::<AST>();
+        let mut page = attempt.target.page_mut(ast);
 
         match &mut page.current_usage {
             Usage::Readonly(ref mut count) => match &attempt.requested_usage {
@@ -408,7 +408,7 @@ impl AddressBook {
 
     #[inline(never)]
     fn cancel<AST: AtScheduleThread>(&mut self, ast: AST, attempt: &mut LockAttempt) {
-        let mut page = attempt.target.page_mut::<AST>();
+        let mut page = attempt.target.page_mut(ast);
 
         match page.next_usage {
             Usage::Unused => {
@@ -653,7 +653,7 @@ fn attempt_lock_for_execution<'a, AST: AtScheduleThread>(
     let mut provisional_count = 0;
 
     for attempt in placeholder_attempts.iter_mut() {
-        AddressBook::attempt_lock_address::<AST>(from_runnable, prefer_immediate, unique_weight, attempt);
+        AddressBook::attempt_lock_address(ast, from_runnable, prefer_immediate, unique_weight, attempt);
         match attempt.status {
             LockStatus::Succeded => {},
             LockStatus::Failed => {
@@ -784,7 +784,8 @@ impl ScheduleStage {
             // plumb message_hash into StatusCache or implmenent our own for duplicate tx
             // detection?
 
-            let (unlockable_count, provisional_count) = attempt_lock_for_execution::<AST>(
+            let (unlockable_count, provisional_count) = attempt_lock_for_execution(
+                ast,
                 from_runnable,
                 prefer_immediate,
                 address_book,
@@ -795,7 +796,8 @@ impl ScheduleStage {
 
             if unlockable_count > 0 {
                 //trace!("reset_lock_for_failed_execution(): {:?} {}", (&unique_weight, from_runnable), next_task.tx.0.signature());
-                Self::reset_lock_for_failed_execution::<AST>(
+                Self::reset_lock_for_failed_execution(
+                    ast,
                     address_book,
                     &unique_weight,
                     &mut next_task.tx.1,
@@ -833,7 +835,8 @@ impl ScheduleStage {
                 next_task.mark_as_uncontended();
                 let tracker = std::sync::Arc::new(ProvisioningTracker::new(provisional_count, TaskInQueue::clone(&a2)));
                 *provisioning_tracker_count = provisioning_tracker_count.checked_add(1).unwrap();
-                Self::finalize_lock_for_provisional_execution::<AST>(
+                Self::finalize_lock_for_provisional_execution(
+                    ast,
                     address_book,
                     next_task,
                     tracker
@@ -870,7 +873,7 @@ impl ScheduleStage {
         for l in next_task.tx.1.iter_mut() {
             match l.status {
                 LockStatus::Provisional => {
-                    l.target.page_mut::<AST>().provisional_task_ids.push(std::sync::Arc::clone(&tracker));
+                    l.target.page_mut(ast).provisional_task_ids.push(std::sync::Arc::clone(&tracker));
                 }
                 LockStatus::Succeded => {
                     // do nothing
@@ -892,16 +895,16 @@ impl ScheduleStage {
         from_runnable: bool,
     ) {
         for l in lock_attempts {
-            address_book.reset_lock::<AST>(l, false);
+            address_book.reset_lock(ast, l, false);
         }
     }
 
     #[inline(never)]
     fn unlock_after_execution<AST: AtScheduleThread>(ast: AST, address_book: &mut AddressBook, lock_attempts: &mut Vec<LockAttempt>, provisioning_tracker_count: &mut usize) {
         for mut l in lock_attempts.into_iter() {
-            let newly_uncontended = address_book.reset_lock::<AST>(&mut l, true);
+            let newly_uncontended = address_book.reset_lock(ast, &mut l, true);
 
-            let mut page = l.target.page_mut::<AST>();
+            let mut page = l.target.page_mut(ast);
             if newly_uncontended && page.next_usage == Usage::Unused {
                 let mut inserted = false;
 
@@ -994,7 +997,7 @@ impl ScheduleStage {
         //*commit_time = commit_time.checked_add(1).unwrap();
 
         // which order for data race free?: unlocking / marking
-        Self::unlock_after_execution::<AST>(address_book, &mut ee.lock_attempts, provisioning_tracker_count);
+        Self::unlock_after_execution(ast, address_book, &mut ee.lock_attempts, provisioning_tracker_count);
         ee.task.mark_as_finished();
 
         // block-wide qos validation will be done here
@@ -1020,7 +1023,7 @@ impl ScheduleStage {
         provisioning_tracker_count: &mut usize,
     ) -> Option<Box<ExecutionEnvironment>> {
         let maybe_ee =
-            Self::pop_from_queue_then_lock::<AST>(task_sender, runnable_queue, address_book, contended_count, prefer_immediate, sequence_time, queue_clock, provisioning_tracker_count)
+            Self::pop_from_queue_then_lock(ast, task_sender, runnable_queue, address_book, contended_count, prefer_immediate, sequence_time, queue_clock, provisioning_tracker_count)
                 .map(|(uw, t,ll)| Self::prepare_scheduled_execution(address_book, uw, t, ll, queue_clock, execute_clock));
         maybe_ee
     }
@@ -1106,7 +1109,7 @@ impl ScheduleStage {
                    }
                    let mut processed_execution_environment = maybe_from_exec.unwrap();
                     executing_queue_count = executing_queue_count.checked_sub(1).unwrap();
-                    Self::commit_completed_execution::<AST>(&mut processed_execution_environment, address_book, &mut execute_clock, &mut provisioning_tracker_count);
+                    Self::commit_completed_execution(ast, &mut processed_execution_environment, address_book, &mut execute_clock, &mut provisioning_tracker_count);
                     to_next_stage.send(processed_execution_environment).unwrap();
                }
                recv(from) -> maybe_from => {
@@ -1132,7 +1135,7 @@ impl ScheduleStage {
                 while (executing_queue_count + provisioning_tracker_count) < max_executing_queue_count {
                     trace!("schedule_once (from: {}, to: {}, runnnable: {}, contended: {}, (immediate+provisional)/max: ({}+{})/{}) active from contended: {}!", from.len(), to_execute_substage.len(), runnable_queue.task_count(), contended_count, executing_queue_count, provisioning_tracker_count, max_executing_queue_count, address_book.uncontended_task_ids.len());
                     let prefer_immediate = provisioning_tracker_count/4 > executing_queue_count;
-                    if let Some(ee) = Self::schedule_next_execution::<AST>(&task_sender, runnable_queue, address_book, &mut contended_count, prefer_immediate, &sequence_time, &mut queue_clock, &mut execute_clock, &mut provisioning_tracker_count) {
+                    if let Some(ee) = Self::schedule_next_execution(ast, &task_sender, runnable_queue, address_book, &mut contended_count, prefer_immediate, &sequence_time, &mut queue_clock, &mut execute_clock, &mut provisioning_tracker_count) {
                         executing_queue_count = executing_queue_count.checked_add(1).unwrap();
                         to_execute_substage.send(ee).unwrap();
                     } else {
@@ -1161,7 +1164,7 @@ impl ScheduleStage {
                         from_exec_len = from_exec_len.checked_sub(1).unwrap();
                         empty_from_exec = from_exec_len == 0;
                         executing_queue_count = executing_queue_count.checked_sub(1).unwrap();
-                        Self::commit_completed_execution::<AST>(&mut processed_execution_environment, address_book, &mut execute_clock, &mut provisioning_tracker_count);
+                        Self::commit_completed_execution(ast, &mut processed_execution_environment, address_book, &mut execute_clock, &mut provisioning_tracker_count);
                         to_next_stage.send(processed_execution_environment).unwrap();
                     }
                     if !empty_from {

From 23edc5c06e7b123e9b741a0e2ad0958a9e2fe544 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 2 Sep 2022 22:42:18 +0900
Subject: [PATCH 1584/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index b636b6ba7e7068..fd0c53755920b2 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -98,7 +98,7 @@ impl ExecutionEnvironment {
 unsafe trait AtScheduleThread {}
 
 impl PageRc {
-    fn page_mut<ST: AtScheduleThread>(&mut self) -> std::cell::RefMut<'_, Page> {
+    fn page_mut<AST: AtScheduleThread>(&mut self, _ast: AST) -> std::cell::RefMut<'_, Page> {
         //unsafe { MyRcInner::get_mut_unchecked(&mut self.0) }
         self.0.borrow_mut()
     }

From 194c124cd87cb7d467f674a63dd45de22ce4e2b0 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 2 Sep 2022 22:42:43 +0900
Subject: [PATCH 1585/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index fd0c53755920b2..bd83ac3a922ab1 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -95,7 +95,7 @@ impl ExecutionEnvironment {
     }
 }
 
-unsafe trait AtScheduleThread {}
+unsafe trait AtScheduleThread: Copy {}
 
 impl PageRc {
     fn page_mut<AST: AtScheduleThread>(&mut self, _ast: AST) -> std::cell::RefMut<'_, Page> {

From 9652411575cb79f47289afc26cf5d7f4d60d7de2 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 2 Sep 2022 22:42:56 +0900
Subject: [PATCH 1586/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index bd83ac3a922ab1..fabdc7017dd468 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1188,7 +1188,7 @@ impl ScheduleStage {
         to_execute_substage: &crossbeam_channel::Sender<Box<ExecutionEnvironment>>,
         maybe_to_next_stage: Option<&crossbeam_channel::Sender<Box<ExecutionEnvironment>>>, // assume nonblocking
     ) {
-        #[derive(Debug)]
+        #[derive(Copy, Debug)]
         struct AtTopOfScheduleThread;
         unsafe impl AtScheduleThread for AtTopOfScheduleThread {}
         //impl !Send for AtTopOfScheduleThread {}

From 720ed9c68ded1712685cee57d684006e94453081 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 2 Sep 2022 22:43:09 +0900
Subject: [PATCH 1587/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index fabdc7017dd468..6f1dac3b284cf4 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1188,7 +1188,7 @@ impl ScheduleStage {
         to_execute_substage: &crossbeam_channel::Sender<Box<ExecutionEnvironment>>,
         maybe_to_next_stage: Option<&crossbeam_channel::Sender<Box<ExecutionEnvironment>>>, // assume nonblocking
     ) {
-        #[derive(Copy, Debug)]
+        #[derive(Clone, Copy, Debug)]
         struct AtTopOfScheduleThread;
         unsafe impl AtScheduleThread for AtTopOfScheduleThread {}
         //impl !Send for AtTopOfScheduleThread {}

From 2c62663f4dc549d02ccbd66432d791804f874999 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 2 Sep 2022 22:44:02 +0900
Subject: [PATCH 1588/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 6f1dac3b284cf4..0afc7d702c19f5 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -98,7 +98,7 @@ impl ExecutionEnvironment {
 unsafe trait AtScheduleThread: Copy {}
 
 impl PageRc {
-    fn page_mut<AST: AtScheduleThread>(&mut self, _ast: AST) -> std::cell::RefMut<'_, Page> {
+    fn page_mut<AST: AtScheduleThread>(&self, _ast: AST) -> std::cell::RefMut<'_, Page> {
         //unsafe { MyRcInner::get_mut_unchecked(&mut self.0) }
         self.0.borrow_mut()
     }

From c53a0765dc6664fe6fb00216c99451f8c3ccd83b Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 2 Sep 2022 22:44:52 +0900
Subject: [PATCH 1589/3199] save

---
 scheduler/src/lib.rs | 1 -
 1 file changed, 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 0afc7d702c19f5..e0e6d99adbbee5 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -26,7 +26,6 @@ type MyRcInner = std::sync::Arc<std::cell::RefCell<Page>>;
 
 #[derive(Debug, Clone)]
 pub struct PageRc(MyRcInner);
-unsafe impl Send for PageRc {}
 unsafe impl Sync for PageRc {}
 
 

From 9a64044d7a268cc4b6838ade758fbfd191eb92d8 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 2 Sep 2022 22:45:03 +0900
Subject: [PATCH 1590/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index e0e6d99adbbee5..2fd03deba95b1d 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -26,7 +26,7 @@ type MyRcInner = std::sync::Arc<std::cell::RefCell<Page>>;
 
 #[derive(Debug, Clone)]
 pub struct PageRc(MyRcInner);
-unsafe impl Sync for PageRc {}
+unsafe impl Send for PageRc {}
 
 
 #[derive(Debug)]

From 831cf354602635921ecfa6e2e534a9d40d5e1257 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 2 Sep 2022 22:45:15 +0900
Subject: [PATCH 1591/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 2fd03deba95b1d..e0e6d99adbbee5 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -26,7 +26,7 @@ type MyRcInner = std::sync::Arc<std::cell::RefCell<Page>>;
 
 #[derive(Debug, Clone)]
 pub struct PageRc(MyRcInner);
-unsafe impl Send for PageRc {}
+unsafe impl Sync for PageRc {}
 
 
 #[derive(Debug)]

From 40db77387c7005484ed46c23d3ff34af0bdb2945 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 2 Sep 2022 22:45:24 +0900
Subject: [PATCH 1592/3199] save

---
 scheduler/src/lib.rs | 1 +
 1 file changed, 1 insertion(+)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index e0e6d99adbbee5..0afc7d702c19f5 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -26,6 +26,7 @@ type MyRcInner = std::sync::Arc<std::cell::RefCell<Page>>;
 
 #[derive(Debug, Clone)]
 pub struct PageRc(MyRcInner);
+unsafe impl Send for PageRc {}
 unsafe impl Sync for PageRc {}
 
 

From 4529fa89c505bf35a81a0a5b535fef4c349f3993 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 2 Sep 2022 22:46:04 +0900
Subject: [PATCH 1593/3199] save

---
 scheduler/src/lib.rs | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 0afc7d702c19f5..82b610ad94ab76 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -149,8 +149,7 @@ impl LockAttempt {
     }
 
     pub fn contended_unique_weights(&self) -> &TaskIds {
-        //&self.target.page_ref().contended_unique_weights
-        panic!()
+        &self.target.page_ref().contended_unique_weights
     }
 }
 

From a2971b794d84466cff62db63fc48018799d57923 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 2 Sep 2022 22:46:32 +0900
Subject: [PATCH 1594/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 82b610ad94ab76..8efb4c02c1fbe5 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -149,7 +149,7 @@ impl LockAttempt {
     }
 
     pub fn contended_unique_weights(&self) -> &TaskIds {
-        &self.target.page_ref().contended_unique_weights
+        self.target.page_ref().contended_unique_weights
     }
 }
 

From 8db15306ae71098128c98be5655ff405eb2158fd Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 2 Sep 2022 22:48:22 +0900
Subject: [PATCH 1595/3199] save

---
 scheduler/src/lib.rs | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 8efb4c02c1fbe5..0afc7d702c19f5 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -149,7 +149,8 @@ impl LockAttempt {
     }
 
     pub fn contended_unique_weights(&self) -> &TaskIds {
-        self.target.page_ref().contended_unique_weights
+        //&self.target.page_ref().contended_unique_weights
+        panic!()
     }
 }
 

From 80e5a527e4eaf04c820ca67616313a80b4915667 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 2 Sep 2022 22:50:40 +0900
Subject: [PATCH 1596/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 0afc7d702c19f5..914ad7771a16b3 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -100,7 +100,7 @@ unsafe trait AtScheduleThread: Copy {}
 impl PageRc {
     fn page_mut<AST: AtScheduleThread>(&self, _ast: AST) -> std::cell::RefMut<'_, Page> {
         //unsafe { MyRcInner::get_mut_unchecked(&mut self.0) }
-        self.0.borrow_mut()
+        self.0.borrow_mut().map()
     }
 
     fn page_ref(&self) -> std::cell::Ref<'_, Page> {

From f077e3c626fc103efa382ffdb63c0022eff185c8 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 2 Sep 2022 22:51:30 +0900
Subject: [PATCH 1597/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 914ad7771a16b3..0afc7d702c19f5 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -100,7 +100,7 @@ unsafe trait AtScheduleThread: Copy {}
 impl PageRc {
     fn page_mut<AST: AtScheduleThread>(&self, _ast: AST) -> std::cell::RefMut<'_, Page> {
         //unsafe { MyRcInner::get_mut_unchecked(&mut self.0) }
-        self.0.borrow_mut().map()
+        self.0.borrow_mut()
     }
 
     fn page_ref(&self) -> std::cell::Ref<'_, Page> {

From fa8eba34ad922a1c26bb166473dcbab68893a428 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 2 Sep 2022 22:52:56 +0900
Subject: [PATCH 1598/3199] save

---
 scheduler/src/lib.rs | 1 -
 1 file changed, 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 0afc7d702c19f5..99bb72bedbdc7f 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -217,7 +217,6 @@ impl TaskIds {
 pub struct Page {
     current_usage: Usage,
     next_usage: Usage,
-    contended_unique_weights: TaskIds,
     provisional_task_ids: Vec<std::sync::Arc<ProvisioningTracker>>,
     //loaded account from Accounts db
     //comulative_cu for qos; i.e. track serialized cumulative keyed by addresses and bail out block

From c43bc4c49b5a69b17e743f0eb329a9b16b4129ec Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 2 Sep 2022 22:53:06 +0900
Subject: [PATCH 1599/3199] save

---
 scheduler/src/lib.rs | 1 -
 1 file changed, 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 99bb72bedbdc7f..c4121155f49742 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -228,7 +228,6 @@ impl Page {
         Self {
             current_usage,
             next_usage: Usage::Unused,
-            contended_unique_weights: Default::default(),
             provisional_task_ids: Default::default(),
         }
     }

From 8f33dd85ee60bdf95e228ef378047adf17ecef83 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 2 Sep 2022 22:53:41 +0900
Subject: [PATCH 1600/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index c4121155f49742..77e25cd341782a 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -22,7 +22,7 @@ type MyRcInner<T> = std::rc::Rc<T>;
 unsafe impl Send for PageRc {}
 */
 
-type MyRcInner = std::sync::Arc<std::cell::RefCell<Page>>;
+type MyRcInner = std::sync::Arc<(std::cell::RefCell<Page>, usize)>;
 
 #[derive(Debug, Clone)]
 pub struct PageRc(MyRcInner);

From a7e8bb10c47282e14c39ac6ce64b0e4d562a6495 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 2 Sep 2022 22:54:19 +0900
Subject: [PATCH 1601/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 77e25cd341782a..4773d3d1f52cf1 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -431,7 +431,7 @@ pub struct Preloader {
 impl Preloader {
     #[inline(never)]
     pub fn load(&self, address: Pubkey) -> PageRc {
-        PageRc::clone(&self.book.entry(address).or_insert_with(|| PageRc(MyRcInner::new(core::cell::RefCell::new(Page::new(Usage::unused()))))))
+        PageRc::clone(&self.book.entry(address).or_insert_with(|| PageRc(MyRcInner::new(core::cell::RefCell::new((Page::new(Usage::unused()), Default::default()))))))
     }
 }
 

From 4deb54a086ea90eb3862d77f2acd90e746640160 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 2 Sep 2022 22:55:15 +0900
Subject: [PATCH 1602/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 4773d3d1f52cf1..d39d75ccb6e93f 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -431,7 +431,7 @@ pub struct Preloader {
 impl Preloader {
     #[inline(never)]
     pub fn load(&self, address: Pubkey) -> PageRc {
-        PageRc::clone(&self.book.entry(address).or_insert_with(|| PageRc(MyRcInner::new(core::cell::RefCell::new((Page::new(Usage::unused()), Default::default()))))))
+        PageRc::clone(&self.book.entry(address).or_insert_with(|| PageRc(MyRcInner::new(core::cell::RefCell::new((Page::new(Usage::unused())), Default::default())))))
     }
 }
 

From 1cf563de7b1af0a949b769f60e0856c8c0fcc5f1 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 2 Sep 2022 22:55:38 +0900
Subject: [PATCH 1603/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index d39d75ccb6e93f..4773d3d1f52cf1 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -431,7 +431,7 @@ pub struct Preloader {
 impl Preloader {
     #[inline(never)]
     pub fn load(&self, address: Pubkey) -> PageRc {
-        PageRc::clone(&self.book.entry(address).or_insert_with(|| PageRc(MyRcInner::new(core::cell::RefCell::new((Page::new(Usage::unused())), Default::default())))))
+        PageRc::clone(&self.book.entry(address).or_insert_with(|| PageRc(MyRcInner::new(core::cell::RefCell::new((Page::new(Usage::unused()), Default::default()))))))
     }
 }
 

From 9ae5fec8789c78d794ed7ff5e233ef54aec711e5 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 2 Sep 2022 22:56:09 +0900
Subject: [PATCH 1604/3199] save

---
 scheduler/src/lib.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 4773d3d1f52cf1..19c15d634f6e47 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -100,12 +100,12 @@ unsafe trait AtScheduleThread: Copy {}
 impl PageRc {
     fn page_mut<AST: AtScheduleThread>(&self, _ast: AST) -> std::cell::RefMut<'_, Page> {
         //unsafe { MyRcInner::get_mut_unchecked(&mut self.0) }
-        self.0.borrow_mut()
+        self.0.0.borrow_mut()
     }
 
     fn page_ref(&self) -> std::cell::Ref<'_, Page> {
         //<MyRcInner as std::borrow::Borrow<_>>::borrow(&self.0)
-        self.0.borrow()
+        self.0.0.borrow()
         //&*(self.0)
     }
 }

From a182ab94e1254e04fff29269616a4aa5227e75f2 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 2 Sep 2022 22:57:31 +0900
Subject: [PATCH 1605/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 19c15d634f6e47..169e9acb321fc1 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -431,7 +431,7 @@ pub struct Preloader {
 impl Preloader {
     #[inline(never)]
     pub fn load(&self, address: Pubkey) -> PageRc {
-        PageRc::clone(&self.book.entry(address).or_insert_with(|| PageRc(MyRcInner::new(core::cell::RefCell::new((Page::new(Usage::unused()), Default::default()))))))
+        panic!() //PageRc::clone(&self.book.entry(address).or_insert_with(|| PageRc(MyRcInner::new((core::cell::RefCell::new((Page::new(Usage::unused()), Default::default()))))))
     }
 }
 

From 49f86147dfd104c7dfb29e66142cc18975c8629d Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 2 Sep 2022 22:58:34 +0900
Subject: [PATCH 1606/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 169e9acb321fc1..8b4ade953f9f15 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -22,7 +22,7 @@ type MyRcInner<T> = std::rc::Rc<T>;
 unsafe impl Send for PageRc {}
 */
 
-type MyRcInner = std::sync::Arc<(std::cell::RefCell<Page>, usize)>;
+type MyRcInner = std::sync::Arc<(std::cell::RefCell<Page>, TaskIds)>;
 
 #[derive(Debug, Clone)]
 pub struct PageRc(MyRcInner);

From e7591e81ba327aaa38ef752cb0bd542eaad65644 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 2 Sep 2022 22:59:10 +0900
Subject: [PATCH 1607/3199] save

---
 scheduler/src/lib.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 8b4ade953f9f15..51b2b7eb73499e 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -149,8 +149,8 @@ impl LockAttempt {
     }
 
     pub fn contended_unique_weights(&self) -> &TaskIds {
-        //&self.target.page_ref().contended_unique_weights
-        panic!()
+        &self.target.0.1
+        //panic!()
     }
 }
 

From cc2e1554758391cd9c86e9437a3fb2337adac66b Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 2 Sep 2022 22:59:25 +0900
Subject: [PATCH 1608/3199] save

---
 scheduler/src/lib.rs | 1 -
 1 file changed, 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 51b2b7eb73499e..7e930d6e93a9a7 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -150,7 +150,6 @@ impl LockAttempt {
 
     pub fn contended_unique_weights(&self) -> &TaskIds {
         &self.target.0.1
-        //panic!()
     }
 }
 

From f592f57ed0b5669e092a5e8d4c32dab708ff44ce Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 3 Sep 2022 09:09:06 +0900
Subject: [PATCH 1609/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 7e930d6e93a9a7..e172f327ba10e3 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -430,7 +430,7 @@ pub struct Preloader {
 impl Preloader {
     #[inline(never)]
     pub fn load(&self, address: Pubkey) -> PageRc {
-        panic!() //PageRc::clone(&self.book.entry(address).or_insert_with(|| PageRc(MyRcInner::new((core::cell::RefCell::new((Page::new(Usage::unused()), Default::default()))))))
+        PageRc::clone(&self.book.entry(address).or_insert_with(|| PageRc(MyRcInner::new((core::cell::RefCell::new((Page::new(Usage::unused()), Default::default()))))))
     }
 }
 

From da57b9a92d02c9700b5a9235116f26aa5013e627 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 3 Sep 2022 09:09:17 +0900
Subject: [PATCH 1610/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index e172f327ba10e3..dacca282242f50 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -430,7 +430,7 @@ pub struct Preloader {
 impl Preloader {
     #[inline(never)]
     pub fn load(&self, address: Pubkey) -> PageRc {
-        PageRc::clone(&self.book.entry(address).or_insert_with(|| PageRc(MyRcInner::new((core::cell::RefCell::new((Page::new(Usage::unused()), Default::default()))))))
+        PageRc::clone(&self.book.entry(address).or_insert_with(|| PageRc(MyRcInner::new((core::cell::RefCell::new((Page::new(Usage::unused()), Default::default())))))))
     }
 }
 

From 1eac588e9038851306e58965400bf8dd49f15b10 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 3 Sep 2022 09:09:34 +0900
Subject: [PATCH 1611/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index dacca282242f50..6490350a49774c 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -430,7 +430,7 @@ pub struct Preloader {
 impl Preloader {
     #[inline(never)]
     pub fn load(&self, address: Pubkey) -> PageRc {
-        PageRc::clone(&self.book.entry(address).or_insert_with(|| PageRc(MyRcInner::new((core::cell::RefCell::new((Page::new(Usage::unused()), Default::default())))))))
+        PageRc::clone(&self.book.entry(address).or_insert_with(|| PageRc(MyRcInner::new((core::cell::RefCell::new(Page::new(Usage::unused())), Default::default())))))))
     }
 }
 

From 4a33aad92b2a38ed561ce0a2c09cab68b3847796 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 3 Sep 2022 09:10:02 +0900
Subject: [PATCH 1612/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 6490350a49774c..c8491ad21c15b3 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -430,7 +430,7 @@ pub struct Preloader {
 impl Preloader {
     #[inline(never)]
     pub fn load(&self, address: Pubkey) -> PageRc {
-        PageRc::clone(&self.book.entry(address).or_insert_with(|| PageRc(MyRcInner::new((core::cell::RefCell::new(Page::new(Usage::unused())), Default::default())))))))
+        PageRc::clone(&self.book.entry(address).or_insert_with(|| PageRc(MyRcInner::new((core::cell::RefCell::new(Page::new(Usage::unused())), Default::default())))))
     }
 }
 

From 2cf46ccb05958b2645a66958d13ab67ef586211e Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 3 Sep 2022 09:11:18 +0900
Subject: [PATCH 1613/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index c8491ad21c15b3..f0e85471a8054b 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -463,7 +463,7 @@ struct Bundle {
 #[derive(Debug)]
 pub struct Task {
     unique_weight: UniqueWeight,
-    pub tx: (SanitizedTransaction, Vec<LockAttempt>), // actually should be Bundle
+    pub tx: (SanitizedTransaction, std::cell::RefCell<Vec<LockAttempt>>), // actually should be Bundle
     pub contention_count: usize,
     pub uncontended: std::sync::atomic::AtomicUsize,
     pub sequence_time: std::sync::atomic::AtomicUsize,

From 80091c85dacec6cb26f53217274c7cf17cb7936e Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 3 Sep 2022 09:13:10 +0900
Subject: [PATCH 1614/3199] save

---
 scheduler/src/lib.rs | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index f0e85471a8054b..efe10b9591a57b 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -463,7 +463,7 @@ struct Bundle {
 #[derive(Debug)]
 pub struct Task {
     unique_weight: UniqueWeight,
-    pub tx: (SanitizedTransaction, std::cell::RefCell<Vec<LockAttempt>>), // actually should be Bundle
+    pub tx: (SanitizedTransaction, (Vec<LockAttempt>, usize)), // actually should be Bundle
     pub contention_count: usize,
     pub uncontended: std::sync::atomic::AtomicUsize,
     pub sequence_time: std::sync::atomic::AtomicUsize,
@@ -497,6 +497,10 @@ impl Task {
         })
     }
 
+    fn lock_attempts(&mut self) -> &mut Vec<LockAttempt> {
+        &mut self.tx.1
+    }
+
     pub fn record_sequence_time(&self, clock: usize) {
         //self.sequence_time.store(clock, std::sync::atomic::Ordering::SeqCst);
     }

From 972f23baceadd35d9df63614d3e6b6fc5b36f5c3 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 3 Sep 2022 09:13:38 +0900
Subject: [PATCH 1615/3199] save

---
 scheduler/src/lib.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index efe10b9591a57b..2ee4bfde86d551 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -498,7 +498,7 @@ impl Task {
     }
 
     fn lock_attempts(&mut self) -> &mut Vec<LockAttempt> {
-        &mut self.tx.1
+        &mut self.tx.1.0
     }
 
     pub fn record_sequence_time(&self, clock: usize) {
@@ -1064,7 +1064,7 @@ impl ScheduleStage {
         } else {
             let h = std::thread::Builder::new().name("sol-reaper".to_string()).spawn(move || {
                 while let Ok(mut a) = ee_receiver.recv() {
-                    assert!(a.task.tx.1.is_empty());
+                    assert!(a.task.lock_attempts().is_empty());
                     //assert!(a.task.sequence_time() != usize::max_value());
                     //let lock_attempts = std::mem::take(&mut a.lock_attempts);
                     //drop(lock_attempts);

From 964930151a6704b47ec445c1df0a7926990baee7 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 3 Sep 2022 09:13:52 +0900
Subject: [PATCH 1616/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 2ee4bfde86d551..bfb7de2faa4735 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -871,7 +871,7 @@ impl ScheduleStage {
         next_task: &mut Task,
         tracker: std::sync::Arc<ProvisioningTracker>,
     ) {
-        for l in next_task.tx.1.iter_mut() {
+        for l in next_task.lock_attempts().iter_mut() {
             match l.status {
                 LockStatus::Provisional => {
                     l.target.page_mut(ast).provisional_task_ids.push(std::sync::Arc::clone(&tracker));

From 6a550d3536e7befbd1dfc1f9995185d5ae9b7a6b Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 3 Sep 2022 09:14:16 +0900
Subject: [PATCH 1617/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index bfb7de2faa4735..1376da8ffa1562 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -853,7 +853,7 @@ impl ScheduleStage {
                 *contended_count = contended_count.checked_sub(1).unwrap();
                 next_task.mark_as_uncontended();
             }
-            let lock_attempts = std::mem::take(&mut next_task.tx.1);
+            let lock_attempts = std::mem::take(&mut next_task.lock_attempts());
 
             return Some((unique_weight, arc_next_task, lock_attempts));
         } else {

From b964363f0d1401b276b0ccc17ff5d254efb54d14 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 3 Sep 2022 09:14:29 +0900
Subject: [PATCH 1618/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 1376da8ffa1562..4180ec5a54a00b 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -853,7 +853,7 @@ impl ScheduleStage {
                 *contended_count = contended_count.checked_sub(1).unwrap();
                 next_task.mark_as_uncontended();
             }
-            let lock_attempts = std::mem::take(&mut next_task.lock_attempts());
+            let lock_attempts = std::mem::take(next_task.lock_attempts());
 
             return Some((unique_weight, arc_next_task, lock_attempts));
         } else {

From 45d7fccf531be6bc6fd29fc4ca94290d510ec4d5 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 3 Sep 2022 09:14:46 +0900
Subject: [PATCH 1619/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 4180ec5a54a00b..ea75074cfe043f 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -830,7 +830,7 @@ impl ScheduleStage {
             } else if provisional_count > 0 {
                 assert!(!from_runnable);
                 assert_eq!(unlockable_count, 0);
-                let lock_count = next_task.tx.1.len();
+                let lock_count = next_task.lock_attempts().len();
                 trace!("provisional exec: [{}/{}]", provisional_count, lock_count);
                 *contended_count = contended_count.checked_sub(1).unwrap();
                 next_task.mark_as_uncontended();

From cbe96b24b08cc74854b743b755ea28b5b3fb3116 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 3 Sep 2022 09:14:59 +0900
Subject: [PATCH 1620/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index ea75074cfe043f..8ad3b5c3a9d39e 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -804,7 +804,7 @@ impl ScheduleStage {
                     &mut next_task.tx.1,
                     from_runnable,
                 );
-                let lock_count = next_task.tx.1.len();
+                let lock_count = next_task.lock_attempts().len();
                 next_task.contention_count += 1;
 
                 if from_runnable {

From 2ed22056fab79bf7964838879214f2a1e1f7d9b1 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 3 Sep 2022 09:15:21 +0900
Subject: [PATCH 1621/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 8ad3b5c3a9d39e..079ac4051c3aac 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -801,7 +801,7 @@ impl ScheduleStage {
                     ast,
                     address_book,
                     &unique_weight,
-                    &mut next_task.tx.1,
+                    next_task.lock_attempts(),
                     from_runnable,
                 );
                 let lock_count = next_task.lock_attempts().len();

From 9ad742d0cbeca63188e5c551819c1722d6aa8c89 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 3 Sep 2022 09:15:41 +0900
Subject: [PATCH 1622/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 079ac4051c3aac..c092e94d4f17c8 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -792,7 +792,7 @@ impl ScheduleStage {
                 address_book,
                 &unique_weight,
                 &message_hash,
-                &mut next_task.tx.1,
+                next_task.lock_attempts(),
             );
 
             if unlockable_count > 0 {

From 77c6ff0b3f71fed6f2f52b98d7acc9964bfaf63f Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 3 Sep 2022 09:16:00 +0900
Subject: [PATCH 1623/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index c092e94d4f17c8..cd9e23b371f18a 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -764,7 +764,7 @@ impl ScheduleStage {
             trace!("expediate pop from provisional queue [rest: {}]", address_book.fulfilled_provisional_task_ids.len());
             let next_task: &mut TaskInQueue = unsafe { panic!(); /*TaskInQueue::get_mut_unchecked(&mut a.1)*/ };
 
-            let lock_attempts = std::mem::take(&mut next_task.tx.1);
+            let lock_attempts = std::mem::take(next_task.lock_attempts());
 
             return Some((a.0, a.1, lock_attempts));
         }

From 36c1300e0b780fcbf23aeabcd1f77a4ba3996cd1 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 3 Sep 2022 09:16:21 +0900
Subject: [PATCH 1624/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index cd9e23b371f18a..23058cfc990179 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -565,7 +565,7 @@ impl Task {
         Self {
             unique_weight: self.unique_weight,
             for_indexer: self.tx.1.iter().map(|a| a.clone_for_test()).collect(),
-            tx: (self.tx.0.clone(), self.tx.1.iter().map(|l| l.clone_for_test()).collect::<Vec<_>>()),
+            tx: (self.tx.0.clone(), self.lock_attempts().iter().map(|l| l.clone_for_test()).collect::<Vec<_>>()),
             contention_count: Default::default(),
             uncontended: Default::default(),
             sequence_time: std::sync::atomic::AtomicUsize::new(usize::max_value()),

From 7620b1ad364deba3892f82d9442628da2735a063 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 3 Sep 2022 09:16:47 +0900
Subject: [PATCH 1625/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 23058cfc990179..da146c39a0813d 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -564,7 +564,7 @@ impl Task {
     pub fn clone_for_test(&self) -> Self {
         Self {
             unique_weight: self.unique_weight,
-            for_indexer: self.tx.1.iter().map(|a| a.clone_for_test()).collect(),
+            for_indexer: self.lock_attempts().iter().map(|a| a.clone_for_test()).collect(),
             tx: (self.tx.0.clone(), self.lock_attempts().iter().map(|l| l.clone_for_test()).collect::<Vec<_>>()),
             contention_count: Default::default(),
             uncontended: Default::default(),

From 63906708eb8f28f52c69b0fff3e3228fb2239c2b Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 3 Sep 2022 09:17:20 +0900
Subject: [PATCH 1626/3199] save

---
 scheduler/src/lib.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index da146c39a0813d..e5ba29538a29c4 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -463,7 +463,7 @@ struct Bundle {
 #[derive(Debug)]
 pub struct Task {
     unique_weight: UniqueWeight,
-    pub tx: (SanitizedTransaction, (Vec<LockAttempt>, usize)), // actually should be Bundle
+    pub tx: (SanitizedTransaction, Vec<LockAttempt>), // actually should be Bundle
     pub contention_count: usize,
     pub uncontended: std::sync::atomic::AtomicUsize,
     pub sequence_time: std::sync::atomic::AtomicUsize,
@@ -498,7 +498,7 @@ impl Task {
     }
 
     fn lock_attempts(&mut self) -> &mut Vec<LockAttempt> {
-        &mut self.tx.1.0
+        &mut self.tx.1
     }
 
     pub fn record_sequence_time(&self, clock: usize) {

From fa33d78bf68d0ae01774c12195e9bea57a9bf607 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 3 Sep 2022 09:18:01 +0900
Subject: [PATCH 1627/3199] save

---
 scheduler/src/lib.rs | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index e5ba29538a29c4..1da354f95bf40b 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -501,6 +501,10 @@ impl Task {
         &mut self.tx.1
     }
 
+    fn lock_attempts_not_mut(&self) -> &Vec<LockAttempt> {
+        &self.tx.1
+    }
+
     pub fn record_sequence_time(&self, clock: usize) {
         //self.sequence_time.store(clock, std::sync::atomic::Ordering::SeqCst);
     }
@@ -1064,7 +1068,7 @@ impl ScheduleStage {
         } else {
             let h = std::thread::Builder::new().name("sol-reaper".to_string()).spawn(move || {
                 while let Ok(mut a) = ee_receiver.recv() {
-                    assert!(a.task.lock_attempts().is_empty());
+                    assert!(a.task.lock_attempts_not_mut().is_empty());
                     //assert!(a.task.sequence_time() != usize::max_value());
                     //let lock_attempts = std::mem::take(&mut a.lock_attempts);
                     //drop(lock_attempts);

From b9749550829d127408da6082a037ed1c1e489650 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 3 Sep 2022 09:18:21 +0900
Subject: [PATCH 1628/3199] save

---
 scheduler/src/lib.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 1da354f95bf40b..d5f293652ec746 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -568,8 +568,8 @@ impl Task {
     pub fn clone_for_test(&self) -> Self {
         Self {
             unique_weight: self.unique_weight,
-            for_indexer: self.lock_attempts().iter().map(|a| a.clone_for_test()).collect(),
-            tx: (self.tx.0.clone(), self.lock_attempts().iter().map(|l| l.clone_for_test()).collect::<Vec<_>>()),
+            for_indexer: self.lock_attempts_not_mut().iter().map(|a| a.clone_for_test()).collect(),
+            tx: (self.tx.0.clone(), self.lock_attempts_not_mut().iter().map(|l| l.clone_for_test()).collect::<Vec<_>>()),
             contention_count: Default::default(),
             uncontended: Default::default(),
             sequence_time: std::sync::atomic::AtomicUsize::new(usize::max_value()),

From 0815df2b6124a864d9b68ef3f118b5f4af871281 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 3 Sep 2022 09:19:34 +0900
Subject: [PATCH 1629/3199] save

---
 scheduler/src/lib.rs | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index d5f293652ec746..7810110508f4dd 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -497,7 +497,7 @@ impl Task {
         })
     }
 
-    fn lock_attempts(&mut self) -> &mut Vec<LockAttempt> {
+    fn lock_attempts_mut(&mut self) -> &mut Vec<LockAttempt> {
         &mut self.tx.1
     }
 
@@ -768,7 +768,7 @@ impl ScheduleStage {
             trace!("expediate pop from provisional queue [rest: {}]", address_book.fulfilled_provisional_task_ids.len());
             let next_task: &mut TaskInQueue = unsafe { panic!(); /*TaskInQueue::get_mut_unchecked(&mut a.1)*/ };
 
-            let lock_attempts = std::mem::take(next_task.lock_attempts());
+            let lock_attempts = std::mem::take(next_task.lock_attempts_mut());
 
             return Some((a.0, a.1, lock_attempts));
         }
@@ -796,7 +796,7 @@ impl ScheduleStage {
                 address_book,
                 &unique_weight,
                 &message_hash,
-                next_task.lock_attempts(),
+                next_task.lock_attempts_mut(),
             );
 
             if unlockable_count > 0 {
@@ -805,10 +805,10 @@ impl ScheduleStage {
                     ast,
                     address_book,
                     &unique_weight,
-                    next_task.lock_attempts(),
+                    next_task.lock_attempts_mut(),
                     from_runnable,
                 );
-                let lock_count = next_task.lock_attempts().len();
+                let lock_count = next_task.lock_attempts_mut().len();
                 next_task.contention_count += 1;
 
                 if from_runnable {
@@ -834,7 +834,7 @@ impl ScheduleStage {
             } else if provisional_count > 0 {
                 assert!(!from_runnable);
                 assert_eq!(unlockable_count, 0);
-                let lock_count = next_task.lock_attempts().len();
+                let lock_count = next_task.lock_attempts_mut().len();
                 trace!("provisional exec: [{}/{}]", provisional_count, lock_count);
                 *contended_count = contended_count.checked_sub(1).unwrap();
                 next_task.mark_as_uncontended();
@@ -857,7 +857,7 @@ impl ScheduleStage {
                 *contended_count = contended_count.checked_sub(1).unwrap();
                 next_task.mark_as_uncontended();
             }
-            let lock_attempts = std::mem::take(next_task.lock_attempts());
+            let lock_attempts = std::mem::take(next_task.lock_attempts_mut());
 
             return Some((unique_weight, arc_next_task, lock_attempts));
         } else {
@@ -875,7 +875,7 @@ impl ScheduleStage {
         next_task: &mut Task,
         tracker: std::sync::Arc<ProvisioningTracker>,
     ) {
-        for l in next_task.lock_attempts().iter_mut() {
+        for l in next_task.lock_attempts_mut().iter_mut() {
             match l.status {
                 LockStatus::Provisional => {
                     l.target.page_mut(ast).provisional_task_ids.push(std::sync::Arc::clone(&tracker));

From 44ee2200ddfd35bb743efcda5620f1692c3c249f Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 3 Sep 2022 09:19:55 +0900
Subject: [PATCH 1630/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 7810110508f4dd..e14ed659de99b0 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -36,7 +36,7 @@ pub struct ExecutionEnvironment {
     pub cu: usize,
     pub unique_weight: UniqueWeight,
     pub task: TaskInQueue,
-    pub lock_attempts: Vec<LockAttempt>,
+    pub finalized_lock_attempts: Vec<LockAttempt>,
 }
 
 impl ExecutionEnvironment {

From d46d2983b84f826bac31efd1e2caa80fb194f049 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 3 Sep 2022 09:20:08 +0900
Subject: [PATCH 1631/3199] save

---
 scheduler/src/lib.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index e14ed659de99b0..73090b09fa18f9 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -971,7 +971,7 @@ impl ScheduleStage {
         address_book: &mut AddressBook,
         unique_weight: UniqueWeight,
         task: TaskInQueue,
-        lock_attempts: Vec<LockAttempt>,
+        finalized_lock_attempts: Vec<LockAttempt>,
         queue_clock: &usize,
         execute_clock: &mut usize,
     ) -> Box<ExecutionEnvironment> {
@@ -984,7 +984,7 @@ impl ScheduleStage {
             task,
             unique_weight,
             cu: rng.gen_range(3, 1000),
-            lock_attempts,
+            finalized_lock_attempts,
         })
     }
 

From 2e36ca9e8799b4b3141aed42c779996ee20662f3 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 3 Sep 2022 09:20:19 +0900
Subject: [PATCH 1632/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 73090b09fa18f9..0fca1498c0635f 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1002,7 +1002,7 @@ impl ScheduleStage {
         //*commit_time = commit_time.checked_add(1).unwrap();
 
         // which order for data race free?: unlocking / marking
-        Self::unlock_after_execution(ast, address_book, &mut ee.lock_attempts, provisioning_tracker_count);
+        Self::unlock_after_execution(ast, address_book, &mut ee.finalized_lock_attempts, provisioning_tracker_count);
         ee.task.mark_as_finished();
 
         // block-wide qos validation will be done here

From 34771cab2519bcad78de8593a1358d4bf8dc0ea3 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 3 Sep 2022 09:20:33 +0900
Subject: [PATCH 1633/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 0fca1498c0635f..ea8478feffc743 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -56,7 +56,7 @@ impl ExecutionEnvironment {
         let uq = self.unique_weight;
         //self.task.trace_timestamps("in_exec(self)");
         let should_remove = self.task.contention_count > 0;
-        for mut lock_attempt in self.lock_attempts.iter_mut() {
+        for mut lock_attempt in self.finalized_lock_attempts.iter_mut() {
             let contended_unique_weights = lock_attempt.contended_unique_weights();
             contended_unique_weights.heaviest_task_cursor().map(|mut task_cursor| {
                 let mut found = true;

From c8fc3fd61443fe9ead8e3eb545950492ee387efa Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 3 Sep 2022 09:27:58 +0900
Subject: [PATCH 1634/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index ea8478feffc743..a2a527f6d1878e 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -463,7 +463,7 @@ struct Bundle {
 #[derive(Debug)]
 pub struct Task {
     unique_weight: UniqueWeight,
-    pub tx: (SanitizedTransaction, Vec<LockAttempt>), // actually should be Bundle
+    pub tx: (SanitizedTransaction, std::cell::RefCell<Vec<LockAttempt>>), // actually should be Bundle
     pub contention_count: usize,
     pub uncontended: std::sync::atomic::AtomicUsize,
     pub sequence_time: std::sync::atomic::AtomicUsize,

From 34095160b996c728a7fae66ef07e6ff24ff8c360 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 3 Sep 2022 09:28:42 +0900
Subject: [PATCH 1635/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index a2a527f6d1878e..2fb8a45a35e9e4 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -498,7 +498,7 @@ impl Task {
     }
 
     fn lock_attempts_mut(&mut self) -> &mut Vec<LockAttempt> {
-        &mut self.tx.1
+        self.tx.1.borrow_mut()
     }
 
     fn lock_attempts_not_mut(&self) -> &Vec<LockAttempt> {

From 5af02d7a7c81c61bc58c5f4bcb3895c7e6cfc5fe Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 3 Sep 2022 09:29:30 +0900
Subject: [PATCH 1636/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 2fb8a45a35e9e4..66b34146f7c3d0 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -497,7 +497,7 @@ impl Task {
         })
     }
 
-    fn lock_attempts_mut(&mut self) -> &mut Vec<LockAttempt> {
+    fn lock_attempts_mut(&mut self) -> std::cell::RefMut<'_, Vec<LockAttempt>>/*&mut Vec<LockAttempt>*/ {
         self.tx.1.borrow_mut()
     }
 

From 9c7205e115f22020297910cd1e291e5bde74a5f6 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 3 Sep 2022 09:29:51 +0900
Subject: [PATCH 1637/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 66b34146f7c3d0..a9fc3150c0920e 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -857,7 +857,7 @@ impl ScheduleStage {
                 *contended_count = contended_count.checked_sub(1).unwrap();
                 next_task.mark_as_uncontended();
             }
-            let lock_attempts = std::mem::take(next_task.lock_attempts_mut());
+            let lock_attempts = std::mem::take(&mut next_task.lock_attempts_mut());
 
             return Some((unique_weight, arc_next_task, lock_attempts));
         } else {

From d131fdef51ff6994c25c7eda575cf5c9e6fd94c2 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 3 Sep 2022 09:30:23 +0900
Subject: [PATCH 1638/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index a9fc3150c0920e..d9acb5a6669b67 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -857,7 +857,7 @@ impl ScheduleStage {
                 *contended_count = contended_count.checked_sub(1).unwrap();
                 next_task.mark_as_uncontended();
             }
-            let lock_attempts = std::mem::take(&mut next_task.lock_attempts_mut());
+            let lock_attempts = std::mem::take(&mut *next_task.lock_attempts_mut());
 
             return Some((unique_weight, arc_next_task, lock_attempts));
         } else {

From a383bd5df4fb63145bfa452e7c0c0551066bc14f Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 3 Sep 2022 09:30:47 +0900
Subject: [PATCH 1639/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index d9acb5a6669b67..3f2cbae9ece3ca 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -805,7 +805,7 @@ impl ScheduleStage {
                     ast,
                     address_book,
                     &unique_weight,
-                    next_task.lock_attempts_mut(),
+                    &mut next_task.lock_attempts_mut(),
                     from_runnable,
                 );
                 let lock_count = next_task.lock_attempts_mut().len();

From db343eb1896ed8504c6ff6aefe07d7ae24f28303 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 3 Sep 2022 09:31:07 +0900
Subject: [PATCH 1640/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 3f2cbae9ece3ca..71931fe8561647 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -796,7 +796,7 @@ impl ScheduleStage {
                 address_book,
                 &unique_weight,
                 &message_hash,
-                next_task.lock_attempts_mut(),
+                &mut next_task.lock_attempts_mut(),
             );
 
             if unlockable_count > 0 {

From c091b0ebe14d2696541d705eab9877707710e30f Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 3 Sep 2022 09:31:22 +0900
Subject: [PATCH 1641/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 71931fe8561647..9758ab076637ea 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -768,7 +768,7 @@ impl ScheduleStage {
             trace!("expediate pop from provisional queue [rest: {}]", address_book.fulfilled_provisional_task_ids.len());
             let next_task: &mut TaskInQueue = unsafe { panic!(); /*TaskInQueue::get_mut_unchecked(&mut a.1)*/ };
 
-            let lock_attempts = std::mem::take(next_task.lock_attempts_mut());
+            let lock_attempts = std::mem::take(&mut *next_task.lock_attempts_mut());
 
             return Some((a.0, a.1, lock_attempts));
         }

From d5045b4e7c85b40ecc98aa4624fa9fbe97799d2e Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 3 Sep 2022 09:32:03 +0900
Subject: [PATCH 1642/3199] save

---
 scheduler/src/lib.rs | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 9758ab076637ea..ae1006404eb6d2 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -475,6 +475,9 @@ pub struct Task {
     pub for_indexer: Vec<LockAttempt>,
 }
 
+unsafe impl Send for Task {}
+unsafe impl Sync for Task {}
+
 // sequence_time -> seq clock
 // queue_time -> queue clock
 // execute_time ---> exec clock

From e946a250c8f97ede787ab97242005812a3416a9b Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 3 Sep 2022 09:32:42 +0900
Subject: [PATCH 1643/3199] save

---
 scheduler/src/lib.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index ae1006404eb6d2..8a32c9ebcdaff8 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -504,8 +504,8 @@ impl Task {
         self.tx.1.borrow_mut()
     }
 
-    fn lock_attempts_not_mut(&self) -> &Vec<LockAttempt> {
-        &self.tx.1
+    fn lock_attempts_not_mut(&self) -> std::cell::Ref<'_, Vec<LockAttempt>>/*&Vec<LockAttempt>*/ {
+        self.tx.1.borrow()
     }
 
     pub fn record_sequence_time(&self, clock: usize) {

From 0b1b18a743f6cd3c9a336919933a037c1fb5573a Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 3 Sep 2022 09:33:24 +0900
Subject: [PATCH 1644/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 8a32c9ebcdaff8..543f93fcfe5374 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -488,7 +488,7 @@ impl Task {
         TaskInQueue::new(Self {
             for_indexer: tx.1.iter().map(|a| a.clone_for_test()).collect(),
             unique_weight,
-            tx,
+            tx: (std::cell::RefCell::new(tx.0), tx.1),
             contention_count: 0,
             uncontended: Default::default(),
             sequence_time: std::sync::atomic::AtomicUsize::new(usize::max_value()),

From 98619f28953b0dd19d5c0f3e6b447f9b8ad36ec9 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 3 Sep 2022 09:33:53 +0900
Subject: [PATCH 1645/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 543f93fcfe5374..a7d60ac9b164ed 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -488,7 +488,7 @@ impl Task {
         TaskInQueue::new(Self {
             for_indexer: tx.1.iter().map(|a| a.clone_for_test()).collect(),
             unique_weight,
-            tx: (std::cell::RefCell::new(tx.0), tx.1),
+            tx: (tx.0, std::cell::RefCell::new(tx.1)),
             contention_count: 0,
             uncontended: Default::default(),
             sequence_time: std::sync::atomic::AtomicUsize::new(usize::max_value()),

From 27ce51d2205dadc1ee4071778b67666af70d9d44 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 3 Sep 2022 09:34:21 +0900
Subject: [PATCH 1646/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index a7d60ac9b164ed..684405cf9b92be 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -572,7 +572,7 @@ impl Task {
         Self {
             unique_weight: self.unique_weight,
             for_indexer: self.lock_attempts_not_mut().iter().map(|a| a.clone_for_test()).collect(),
-            tx: (self.tx.0.clone(), self.lock_attempts_not_mut().iter().map(|l| l.clone_for_test()).collect::<Vec<_>>()),
+            tx: (self.tx.0.clone(), self.lock_attempts_not_mut().iter().map(|l| std::cell::RefCell::new(l.clone_for_test())).collect::<Vec<_>>()),
             contention_count: Default::default(),
             uncontended: Default::default(),
             sequence_time: std::sync::atomic::AtomicUsize::new(usize::max_value()),

From 16c7defdf7961007e75773b8a4ef4734cc3eea36 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 3 Sep 2022 09:34:44 +0900
Subject: [PATCH 1647/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 684405cf9b92be..8faad5076669ee 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -572,7 +572,7 @@ impl Task {
         Self {
             unique_weight: self.unique_weight,
             for_indexer: self.lock_attempts_not_mut().iter().map(|a| a.clone_for_test()).collect(),
-            tx: (self.tx.0.clone(), self.lock_attempts_not_mut().iter().map(|l| std::cell::RefCell::new(l.clone_for_test())).collect::<Vec<_>>()),
+            tx: (self.tx.0.clone(), std::cell::RefCell::new(self.lock_attempts_not_mut().iter().map(|l| l.clone_for_test()).collect::<Vec<_>>())),
             contention_count: Default::default(),
             uncontended: Default::default(),
             sequence_time: std::sync::atomic::AtomicUsize::new(usize::max_value()),

From d71cc924640fd2d8e049fee014ef843541a719dd Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 3 Sep 2022 09:35:07 +0900
Subject: [PATCH 1648/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 8faad5076669ee..54b89ff4343af8 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -500,7 +500,7 @@ impl Task {
         })
     }
 
-    fn lock_attempts_mut(&mut self) -> std::cell::RefMut<'_, Vec<LockAttempt>>/*&mut Vec<LockAttempt>*/ {
+    fn lock_attempts_mut(&self) -> std::cell::RefMut<'_, Vec<LockAttempt>>/*&mut Vec<LockAttempt>*/ {
         self.tx.1.borrow_mut()
     }
 

From 4b388e2924389a4ec1fd1137cb3ea5f53e84b6f8 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 3 Sep 2022 09:35:41 +0900
Subject: [PATCH 1649/3199] save

---
 scheduler/src/lib.rs | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 54b89ff4343af8..1e4ebb703e7756 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -769,9 +769,8 @@ impl ScheduleStage {
     ) -> Option<(UniqueWeight, TaskInQueue, Vec<LockAttempt>)> {
         if let Some(mut a) = address_book.fulfilled_provisional_task_ids.pop_last() {
             trace!("expediate pop from provisional queue [rest: {}]", address_book.fulfilled_provisional_task_ids.len());
-            let next_task: &mut TaskInQueue = unsafe { panic!(); /*TaskInQueue::get_mut_unchecked(&mut a.1)*/ };
 
-            let lock_attempts = std::mem::take(&mut *next_task.lock_attempts_mut());
+            let lock_attempts = std::mem::take(&mut *a.1.lock_attempts_mut());
 
             return Some((a.0, a.1, lock_attempts));
         }

From 84dfc26f13432fd0935eec05b2d25d74b6414e34 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 3 Sep 2022 09:36:44 +0900
Subject: [PATCH 1650/3199] save

---
 scheduler/src/lib.rs | 7 ++-----
 1 file changed, 2 insertions(+), 5 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 1e4ebb703e7756..2b661364e280ae 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -99,7 +99,6 @@ unsafe trait AtScheduleThread: Copy {}
 
 impl PageRc {
     fn page_mut<AST: AtScheduleThread>(&self, _ast: AST) -> std::cell::RefMut<'_, Page> {
-        //unsafe { MyRcInner::get_mut_unchecked(&mut self.0) }
         self.0.0.borrow_mut()
     }
 
@@ -777,14 +776,12 @@ impl ScheduleStage {
 
         trace!("pop begin");
         loop {
-        if let Some((from_runnable, mut arc_next_task)) = Self::select_next_task(runnable_queue, address_book) {
+        if let Some((from_runnable, mut next_task)) = Self::select_next_task(runnable_queue, address_book) {
             trace!("pop loop iteration");
             if from_runnable {
-                arc_next_task.record_queue_time(*sequence_clock, *queue_clock);
+                next_task.record_queue_time(*sequence_clock, *queue_clock);
                 *queue_clock = queue_clock.checked_add(1).unwrap();
             }
-            let a2 = TaskInQueue::clone(&arc_next_task);
-            let next_task: &mut TaskInQueue = unsafe { panic!(); /*TaskInQueue::get_mut_unchecked(&mut arc_next_task)*/ };
             let unique_weight = next_task.unique_weight;
             let message_hash = next_task.tx.0.message_hash();
 

From 19dc99f6048d574a7e5730b034390a81b5301853 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 3 Sep 2022 09:37:23 +0900
Subject: [PATCH 1651/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 2b661364e280ae..d6f261c09695e3 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -871,7 +871,7 @@ impl ScheduleStage {
     fn finalize_lock_for_provisional_execution<AST: AtScheduleThread>(
         ast: AST,
         address_book: &mut AddressBook,
-        next_task: &mut Task,
+        next_task: &Task,
         tracker: std::sync::Arc<ProvisioningTracker>,
     ) {
         for l in next_task.lock_attempts_mut().iter_mut() {

From e79e8806118fdc2dc28811e62d0bdc4083892529 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 3 Sep 2022 09:37:35 +0900
Subject: [PATCH 1652/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index d6f261c09695e3..bf4262d386047c 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -842,7 +842,7 @@ impl ScheduleStage {
                 Self::finalize_lock_for_provisional_execution(
                     ast,
                     address_book,
-                    next_task,
+                    &next_task,
                     tracker
                 );
 

From a1bcbdb414e9bc7143ca429b26d4d13a17c8dc06 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 3 Sep 2022 09:37:48 +0900
Subject: [PATCH 1653/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index bf4262d386047c..bd9a6342ce3ddc 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -858,7 +858,7 @@ impl ScheduleStage {
             }
             let lock_attempts = std::mem::take(&mut *next_task.lock_attempts_mut());
 
-            return Some((unique_weight, arc_next_task, lock_attempts));
+            return Some((unique_weight, next_task, lock_attempts));
         } else {
             break;
         }

From 947df845342e39e2020244e8c5b057b9170f98e1 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 3 Sep 2022 09:38:04 +0900
Subject: [PATCH 1654/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index bd9a6342ce3ddc..be67401f2af732 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -837,7 +837,7 @@ impl ScheduleStage {
                 trace!("provisional exec: [{}/{}]", provisional_count, lock_count);
                 *contended_count = contended_count.checked_sub(1).unwrap();
                 next_task.mark_as_uncontended();
-                let tracker = std::sync::Arc::new(ProvisioningTracker::new(provisional_count, TaskInQueue::clone(&a2)));
+                let tracker = std::sync::Arc::new(ProvisioningTracker::new(provisional_count, TaskInQueue::clone(&next_task)));
                 *provisioning_tracker_count = provisioning_tracker_count.checked_add(1).unwrap();
                 Self::finalize_lock_for_provisional_execution(
                     ast,

From e6639c4f9274cd10662b279abc35ecdf910f767e Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 3 Sep 2022 09:38:18 +0900
Subject: [PATCH 1655/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index be67401f2af732..0e1f83310e1da2 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -817,7 +817,7 @@ impl ScheduleStage {
                     //for lock_attempt in next_task.tx.1.iter() {
                     //    lock_attempt.contended_unique_weights().insert_task(unique_weight, TaskInQueue::clone(&a2));
                     //}
-                    task_sender.send((TaskInQueue::clone(&a2), std::mem::take(&mut next_task.for_indexer))).unwrap();
+                    task_sender.send((TaskInQueue::clone(&next_task), std::mem::take(&mut next_task.for_indexer))).unwrap();
 
                     // maybe run lightweight prune logic on contended_queue here.
                 } else {

From 6d914f4898a5cd7aafd414aa0513ccc7de062142 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 3 Sep 2022 09:38:45 +0900
Subject: [PATCH 1656/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 0e1f83310e1da2..7a9af788d01c20 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -817,7 +817,7 @@ impl ScheduleStage {
                     //for lock_attempt in next_task.tx.1.iter() {
                     //    lock_attempt.contended_unique_weights().insert_task(unique_weight, TaskInQueue::clone(&a2));
                     //}
-                    task_sender.send((TaskInQueue::clone(&next_task), std::mem::take(&mut next_task.for_indexer))).unwrap();
+                    panic!(); // task_sender.send((TaskInQueue::clone(&next_task), std::mem::take(&mut next_task.for_indexer))).unwrap();
 
                     // maybe run lightweight prune logic on contended_queue here.
                 } else {

From 24e71ad6cc876993eb7cb3a02b2cd4ba9698f33d Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 3 Sep 2022 09:39:04 +0900
Subject: [PATCH 1657/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 7a9af788d01c20..d1410cae59b143 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -808,7 +808,7 @@ impl ScheduleStage {
                     from_runnable,
                 );
                 let lock_count = next_task.lock_attempts_mut().len();
-                next_task.contention_count += 1;
+                panic!();//next_task.contention_count += 1;
 
                 if from_runnable {
                     trace!("move to contended due to lock failure [{}/{}/{}]", unlockable_count, provisional_count, lock_count);

From 58f8f3aee8b483a762615d7b33e49db0af41b1a7 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 3 Sep 2022 09:39:39 +0900
Subject: [PATCH 1658/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index d1410cae59b143..84f6b3da395979 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -499,7 +499,7 @@ impl Task {
         })
     }
 
-    fn lock_attempts_mut(&self) -> std::cell::RefMut<'_, Vec<LockAttempt>>/*&mut Vec<LockAttempt>*/ {
+    fn lock_attempts_mut<AST: AtScheduleThread>(&self) -> std::cell::RefMut<'_, Vec<LockAttempt>>/*&mut Vec<LockAttempt>*/ {
         self.tx.1.borrow_mut()
     }
 

From 0d9f42fadd938845a0b081e7cce3ce744bc3cd3d Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 3 Sep 2022 09:40:15 +0900
Subject: [PATCH 1659/3199] save

---
 scheduler/src/lib.rs | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 84f6b3da395979..803013e0260797 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -499,7 +499,7 @@ impl Task {
         })
     }
 
-    fn lock_attempts_mut<AST: AtScheduleThread>(&self) -> std::cell::RefMut<'_, Vec<LockAttempt>>/*&mut Vec<LockAttempt>*/ {
+    fn lock_attempts_mut<AST: AtScheduleThread>(&self, _ast: AST) -> std::cell::RefMut<'_, Vec<LockAttempt>>/*&mut Vec<LockAttempt>*/ {
         self.tx.1.borrow_mut()
     }
 
@@ -769,7 +769,7 @@ impl ScheduleStage {
         if let Some(mut a) = address_book.fulfilled_provisional_task_ids.pop_last() {
             trace!("expediate pop from provisional queue [rest: {}]", address_book.fulfilled_provisional_task_ids.len());
 
-            let lock_attempts = std::mem::take(&mut *a.1.lock_attempts_mut());
+            let lock_attempts = std::mem::take(&mut *a.1.lock_attempts_mut(ast));
 
             return Some((a.0, a.1, lock_attempts));
         }
@@ -795,7 +795,7 @@ impl ScheduleStage {
                 address_book,
                 &unique_weight,
                 &message_hash,
-                &mut next_task.lock_attempts_mut(),
+                &mut next_task.lock_attempts_mut(ast),
             );
 
             if unlockable_count > 0 {
@@ -804,10 +804,10 @@ impl ScheduleStage {
                     ast,
                     address_book,
                     &unique_weight,
-                    &mut next_task.lock_attempts_mut(),
+                    &mut next_task.lock_attempts_mut(ast),
                     from_runnable,
                 );
-                let lock_count = next_task.lock_attempts_mut().len();
+                let lock_count = next_task.lock_attempts_mut(ast).len();
                 panic!();//next_task.contention_count += 1;
 
                 if from_runnable {
@@ -833,7 +833,7 @@ impl ScheduleStage {
             } else if provisional_count > 0 {
                 assert!(!from_runnable);
                 assert_eq!(unlockable_count, 0);
-                let lock_count = next_task.lock_attempts_mut().len();
+                let lock_count = next_task.lock_attempts_mut(ast).len();
                 trace!("provisional exec: [{}/{}]", provisional_count, lock_count);
                 *contended_count = contended_count.checked_sub(1).unwrap();
                 next_task.mark_as_uncontended();
@@ -856,7 +856,7 @@ impl ScheduleStage {
                 *contended_count = contended_count.checked_sub(1).unwrap();
                 next_task.mark_as_uncontended();
             }
-            let lock_attempts = std::mem::take(&mut *next_task.lock_attempts_mut());
+            let lock_attempts = std::mem::take(&mut *next_task.lock_attempts_mut(ast));
 
             return Some((unique_weight, next_task, lock_attempts));
         } else {
@@ -874,7 +874,7 @@ impl ScheduleStage {
         next_task: &Task,
         tracker: std::sync::Arc<ProvisioningTracker>,
     ) {
-        for l in next_task.lock_attempts_mut().iter_mut() {
+        for l in next_task.lock_attempts_mut(ast).iter_mut() {
             match l.status {
                 LockStatus::Provisional => {
                     l.target.page_mut(ast).provisional_task_ids.push(std::sync::Arc::clone(&tracker));

From e4463324887492e306e187ef250fe0fd71ce211d Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 3 Sep 2022 09:41:19 +0900
Subject: [PATCH 1660/3199] save

---
 scheduler/src/lib.rs | 2 --
 1 file changed, 2 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 803013e0260797..de431bca07399b 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1,6 +1,4 @@
 #![feature(map_first_last)]
-#![feature(negative_impls)]
-//#![feature(get_mut_unchecked)]
 
 use {
     crossbeam_channel::{bounded, unbounded},

From ab689104dc87e1566b04314d0c71428b94e49e70 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 3 Sep 2022 09:41:34 +0900
Subject: [PATCH 1661/3199] save

---
 scheduler/src/lib.rs | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index de431bca07399b..1390421178ace2 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1,4 +1,6 @@
 #![feature(map_first_last)]
+#![feature(negative_impls)]
+//#![feature(get_mut_unchecked)]
 
 use {
     crossbeam_channel::{bounded, unbounded},
@@ -1193,8 +1195,8 @@ impl ScheduleStage {
         #[derive(Clone, Copy, Debug)]
         struct AtTopOfScheduleThread;
         unsafe impl AtScheduleThread for AtTopOfScheduleThread {}
-        //impl !Send for AtTopOfScheduleThread {}
-        //impl !Sync for AtTopOfScheduleThread {}
+        impl !Send for AtTopOfScheduleThread {}
+        impl !Sync for AtTopOfScheduleThread {}
 
         Self::_run::<AtTopOfScheduleThread>(AtTopOfScheduleThread, max_executing_queue_count, runnable_queue, address_book, from, from_exec, to_execute_substage, maybe_to_next_stage)
     }

From 44402d520d0e54c23c07b26aaef9027cda5c9d30 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 3 Sep 2022 09:41:42 +0900
Subject: [PATCH 1662/3199] save

---
 scheduler/src/lib.rs | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 1390421178ace2..64162528782c29 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1,5 +1,4 @@
 #![feature(map_first_last)]
-#![feature(negative_impls)]
 //#![feature(get_mut_unchecked)]
 
 use {
@@ -1195,8 +1194,6 @@ impl ScheduleStage {
         #[derive(Clone, Copy, Debug)]
         struct AtTopOfScheduleThread;
         unsafe impl AtScheduleThread for AtTopOfScheduleThread {}
-        impl !Send for AtTopOfScheduleThread {}
-        impl !Sync for AtTopOfScheduleThread {}
 
         Self::_run::<AtTopOfScheduleThread>(AtTopOfScheduleThread, max_executing_queue_count, runnable_queue, address_book, from, from_exec, to_execute_substage, maybe_to_next_stage)
     }

From e74139c46fdd83311beabf5151848e639a283392 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 3 Sep 2022 09:44:59 +0900
Subject: [PATCH 1663/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 64162528782c29..3d718159b842a8 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1066,7 +1066,7 @@ impl ScheduleStage {
         } else {
             let h = std::thread::Builder::new().name("sol-reaper".to_string()).spawn(move || {
                 while let Ok(mut a) = ee_receiver.recv() {
-                    assert!(a.task.lock_attempts_not_mut().is_empty());
+                    assert!(a.task.lock_attempts_mut(ast).is_empty());
                     //assert!(a.task.sequence_time() != usize::max_value());
                     //let lock_attempts = std::mem::take(&mut a.lock_attempts);
                     //drop(lock_attempts);

From 9128225a480439fba7285cd8e46da7dfb4e6dea4 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 3 Sep 2022 09:45:07 +0900
Subject: [PATCH 1664/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 3d718159b842a8..64162528782c29 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1066,7 +1066,7 @@ impl ScheduleStage {
         } else {
             let h = std::thread::Builder::new().name("sol-reaper".to_string()).spawn(move || {
                 while let Ok(mut a) = ee_receiver.recv() {
-                    assert!(a.task.lock_attempts_mut(ast).is_empty());
+                    assert!(a.task.lock_attempts_not_mut().is_empty());
                     //assert!(a.task.sequence_time() != usize::max_value());
                     //let lock_attempts = std::mem::take(&mut a.lock_attempts);
                     //drop(lock_attempts);

From e368929a4a631c731c9895ea16afd2a8ce579415 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 3 Sep 2022 09:46:22 +0900
Subject: [PATCH 1665/3199] save

---
 scheduler/src/lib.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 64162528782c29..6e4aa49e8641e3 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -498,11 +498,11 @@ impl Task {
         })
     }
 
-    fn lock_attempts_mut<AST: AtScheduleThread>(&self, _ast: AST) -> std::cell::RefMut<'_, Vec<LockAttempt>>/*&mut Vec<LockAttempt>*/ {
+    fn lock_attempts_mut<AST: AtScheduleThread>(&self, _ast: AST) -> std::cell::RefMut<'_, Vec<LockAttempt>> {
         self.tx.1.borrow_mut()
     }
 
-    fn lock_attempts_not_mut(&self) -> std::cell::Ref<'_, Vec<LockAttempt>>/*&Vec<LockAttempt>*/ {
+    fn lock_attempts_not_mut(&self) -> std::cell::Ref<'_, Vec<LockAttempt>> {
         self.tx.1.borrow()
     }
 

From c8b92ce56d9d34454a90df936fba2c4600a6c2c3 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 3 Sep 2022 09:56:43 +0900
Subject: [PATCH 1666/3199] save

---
 scheduler/src/lib.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 6e4aa49e8641e3..5dfbf98aeca117 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -473,8 +473,8 @@ pub struct Task {
     pub for_indexer: Vec<LockAttempt>,
 }
 
-unsafe impl Send for Task {}
-unsafe impl Sync for Task {}
+//unsafe impl Send for Task {}
+//unsafe impl Sync for Task {}
 
 // sequence_time -> seq clock
 // queue_time -> queue clock

From 7deac054c7d2e280c85141a02c5b003c2f3f2db9 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 3 Sep 2022 09:57:32 +0900
Subject: [PATCH 1667/3199] save

---
 scheduler/src/lib.rs | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 5dfbf98aeca117..1be9cb3eb76330 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -461,7 +461,7 @@ struct Bundle {
 #[derive(Debug)]
 pub struct Task {
     unique_weight: UniqueWeight,
-    pub tx: (SanitizedTransaction, std::cell::RefCell<Vec<LockAttempt>>), // actually should be Bundle
+    pub tx: (SanitizedTransaction, LockAttemptsInCell), // actually should be Bundle
     pub contention_count: usize,
     pub uncontended: std::sync::atomic::AtomicUsize,
     pub sequence_time: std::sync::atomic::AtomicUsize,
@@ -473,8 +473,10 @@ pub struct Task {
     pub for_indexer: Vec<LockAttempt>,
 }
 
-//unsafe impl Send for Task {}
-//unsafe impl Sync for Task {}
+struct LockAttemptsInCell(std::cell::RefCell<Vec<LockAttempt>>);
+
+unsafe impl Send for LockAttemptsInCell {}
+unsafe impl Sync for LockAttemptsInCell {}
 
 // sequence_time -> seq clock
 // queue_time -> queue clock

From dce623e28ff0de3d9b28721c39de6d3ecdc93785 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 3 Sep 2022 09:57:54 +0900
Subject: [PATCH 1668/3199] save

---
 scheduler/src/lib.rs | 1 +
 1 file changed, 1 insertion(+)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 1be9cb3eb76330..9442a4d57246f6 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -473,6 +473,7 @@ pub struct Task {
     pub for_indexer: Vec<LockAttempt>,
 }
 
+#[derive(Debug)]
 struct LockAttemptsInCell(std::cell::RefCell<Vec<LockAttempt>>);
 
 unsafe impl Send for LockAttemptsInCell {}

From dc1e74fc9e98a663cb3d1a90596d07e83de91e8a Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 3 Sep 2022 09:58:11 +0900
Subject: [PATCH 1669/3199] save

---
 scheduler/src/lib.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 9442a4d57246f6..152a9776b8adde 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -502,11 +502,11 @@ impl Task {
     }
 
     fn lock_attempts_mut<AST: AtScheduleThread>(&self, _ast: AST) -> std::cell::RefMut<'_, Vec<LockAttempt>> {
-        self.tx.1.borrow_mut()
+        self.tx.1.0.borrow_mut()
     }
 
     fn lock_attempts_not_mut(&self) -> std::cell::Ref<'_, Vec<LockAttempt>> {
-        self.tx.1.borrow()
+        self.tx.1.0.borrow()
     }
 
     pub fn record_sequence_time(&self, clock: usize) {

From 6e06b9de7bbb8636320a41b826e2a56a8585f409 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 3 Sep 2022 09:59:08 +0900
Subject: [PATCH 1670/3199] save

---
 scheduler/src/lib.rs | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 152a9776b8adde..a6929356681dc5 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -479,6 +479,12 @@ struct LockAttemptsInCell(std::cell::RefCell<Vec<LockAttempt>>);
 unsafe impl Send for LockAttemptsInCell {}
 unsafe impl Sync for LockAttemptsInCell {}
 
+impl LockAttemptsInCell {
+    fn new(ll: std::cell:RefCell<Vec<LockAttempt>>) -> Self {
+        Self(ll)
+    }
+}
+
 // sequence_time -> seq clock
 // queue_time -> queue clock
 // execute_time ---> exec clock

From b09a73f09b54819ce2eaeebb39a5d23c44b5f570 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 3 Sep 2022 09:59:21 +0900
Subject: [PATCH 1671/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index a6929356681dc5..b6804a8884a925 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -480,7 +480,7 @@ unsafe impl Send for LockAttemptsInCell {}
 unsafe impl Sync for LockAttemptsInCell {}
 
 impl LockAttemptsInCell {
-    fn new(ll: std::cell:RefCell<Vec<LockAttempt>>) -> Self {
+    fn new(ll: std::cell::RefCell<Vec<LockAttempt>>) -> Self {
         Self(ll)
     }
 }

From 9d874a285db6df413eb19becbcc70eeab7fb2e7f Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 3 Sep 2022 09:59:46 +0900
Subject: [PATCH 1672/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index b6804a8884a925..8b4d9d274b0e08 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -495,7 +495,7 @@ impl Task {
         TaskInQueue::new(Self {
             for_indexer: tx.1.iter().map(|a| a.clone_for_test()).collect(),
             unique_weight,
-            tx: (tx.0, std::cell::RefCell::new(tx.1)),
+            tx: (tx.0, LockAttemptsInCell::new(std::cell::RefCell::new(tx.1))),
             contention_count: 0,
             uncontended: Default::default(),
             sequence_time: std::sync::atomic::AtomicUsize::new(usize::max_value()),

From b54e7fdcea3b4f9ac45a4d6d1ac94bcab96bff60 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 3 Sep 2022 10:00:09 +0900
Subject: [PATCH 1673/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 8b4d9d274b0e08..af181bdcd0d883 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -579,7 +579,7 @@ impl Task {
         Self {
             unique_weight: self.unique_weight,
             for_indexer: self.lock_attempts_not_mut().iter().map(|a| a.clone_for_test()).collect(),
-            tx: (self.tx.0.clone(), std::cell::RefCell::new(self.lock_attempts_not_mut().iter().map(|l| l.clone_for_test()).collect::<Vec<_>>())),
+            tx: (self.tx.0.clone(), LockAttemptsInCell::new(std::cell::RefCell::new(self.lock_attempts_not_mut().iter().map(|l| l.clone_for_test()).collect::<Vec<_>>()))),
             contention_count: Default::default(),
             uncontended: Default::default(),
             sequence_time: std::sync::atomic::AtomicUsize::new(usize::max_value()),

From ee0a2d13d6b940c3afcf398d0ca518aba41012ab Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 3 Sep 2022 10:00:22 +0900
Subject: [PATCH 1674/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index af181bdcd0d883..51239113c91b8f 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -474,7 +474,7 @@ pub struct Task {
 }
 
 #[derive(Debug)]
-struct LockAttemptsInCell(std::cell::RefCell<Vec<LockAttempt>>);
+pub struct LockAttemptsInCell(std::cell::RefCell<Vec<LockAttempt>>);
 
 unsafe impl Send for LockAttemptsInCell {}
 unsafe impl Sync for LockAttemptsInCell {}

From ef30a8ef5c3a912cdda1e6b0646b68d994c34c0a Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 3 Sep 2022 10:00:37 +0900
Subject: [PATCH 1675/3199] save

---
 scheduler/src/lib.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 51239113c91b8f..1f2de178fd3784 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -476,8 +476,8 @@ pub struct Task {
 #[derive(Debug)]
 pub struct LockAttemptsInCell(std::cell::RefCell<Vec<LockAttempt>>);
 
-unsafe impl Send for LockAttemptsInCell {}
-unsafe impl Sync for LockAttemptsInCell {}
+//unsafe impl Send for LockAttemptsInCell {}
+//unsafe impl Sync for LockAttemptsInCell {}
 
 impl LockAttemptsInCell {
     fn new(ll: std::cell::RefCell<Vec<LockAttempt>>) -> Self {

From 3dc193db0e1d398b03cc631e0f8f5825b79d9a89 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 3 Sep 2022 10:00:46 +0900
Subject: [PATCH 1676/3199] save

---
 scheduler/src/lib.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 1f2de178fd3784..51239113c91b8f 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -476,8 +476,8 @@ pub struct Task {
 #[derive(Debug)]
 pub struct LockAttemptsInCell(std::cell::RefCell<Vec<LockAttempt>>);
 
-//unsafe impl Send for LockAttemptsInCell {}
-//unsafe impl Sync for LockAttemptsInCell {}
+unsafe impl Send for LockAttemptsInCell {}
+unsafe impl Sync for LockAttemptsInCell {}
 
 impl LockAttemptsInCell {
     fn new(ll: std::cell::RefCell<Vec<LockAttempt>>) -> Self {

From 975bfac060b75ca966572f164a0e9588101f1694 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 3 Sep 2022 10:03:10 +0900
Subject: [PATCH 1677/3199] save

---
 scheduler/src/lib.rs | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 51239113c91b8f..b041a9dec02015 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -995,10 +995,6 @@ impl ScheduleStage {
         })
     }
 
-    fn commit_completed_execution2<AST: AtScheduleThread>(_a: AST) {
-        panic!();
-    }
-
     #[inline(never)]
     fn commit_completed_execution<AST: AtScheduleThread>(ast: AST, ee: &mut ExecutionEnvironment, address_book: &mut AddressBook, commit_time: &mut usize, provisioning_tracker_count: &mut usize) {
         // do par()-ly?

From fe27498908d7cccd7be93aacd188939684bc62d2 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 3 Sep 2022 10:04:45 +0900
Subject: [PATCH 1678/3199] save

---
 scheduler/src/lib.rs | 7 +------
 1 file changed, 1 insertion(+), 6 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index b041a9dec02015..e33c719a00c125 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -95,17 +95,12 @@ impl ExecutionEnvironment {
 }
 
 unsafe trait AtScheduleThread: Copy {}
+unsafe trait NotAtScheduleThread: Copy {}
 
 impl PageRc {
     fn page_mut<AST: AtScheduleThread>(&self, _ast: AST) -> std::cell::RefMut<'_, Page> {
         self.0.0.borrow_mut()
     }
-
-    fn page_ref(&self) -> std::cell::Ref<'_, Page> {
-        //<MyRcInner as std::borrow::Borrow<_>>::borrow(&self.0)
-        self.0.0.borrow()
-        //&*(self.0)
-    }
 }
 
 #[derive(Clone, Debug)]

From a12db02b88a0fe819b45f0e9604c954ef4cda24f Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 3 Sep 2022 10:05:16 +0900
Subject: [PATCH 1679/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index e33c719a00c125..9e3eb8a8ae3986 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -506,7 +506,7 @@ impl Task {
         self.tx.1.0.borrow_mut()
     }
 
-    fn lock_attempts_not_mut(&self) -> std::cell::Ref<'_, Vec<LockAttempt>> {
+    fn lock_attempts_not_mut<NAST: NotAtScheduleThread>(&self) -> std::cell::Ref<'_, Vec<LockAttempt>> {
         self.tx.1.0.borrow()
     }
 

From 063fef0b68fe728262f7cfac219c4a95b0a2b67a Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 3 Sep 2022 10:06:14 +0900
Subject: [PATCH 1680/3199] save

---
 scheduler/src/lib.rs | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 9e3eb8a8ae3986..9bb13c24dd2eab 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1065,8 +1065,13 @@ impl ScheduleStage {
             (to_next_stage, None)
         } else {
             let h = std::thread::Builder::new().name("sol-reaper".to_string()).spawn(move || {
+                #[derive(Clone, Copy, Debug)]
+                struct NotAtTopOfScheduleThread;
+                unsafe impl NotAtScheduleThread for NotAtTopOfScheduleThread {}
+                let nast = NotAtScheduleThread;
+
                 while let Ok(mut a) = ee_receiver.recv() {
-                    assert!(a.task.lock_attempts_not_mut().is_empty());
+                    assert!(a.task.lock_attempts_not_mut(nast).is_empty());
                     //assert!(a.task.sequence_time() != usize::max_value());
                     //let lock_attempts = std::mem::take(&mut a.lock_attempts);
                     //drop(lock_attempts);

From 018c7fc6719a5b4290947628eda88908eba908b0 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 3 Sep 2022 10:06:33 +0900
Subject: [PATCH 1681/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 9bb13c24dd2eab..595a2b9a310526 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -506,7 +506,7 @@ impl Task {
         self.tx.1.0.borrow_mut()
     }
 
-    fn lock_attempts_not_mut<NAST: NotAtScheduleThread>(&self) -> std::cell::Ref<'_, Vec<LockAttempt>> {
+    fn lock_attempts_not_mut<NAST: NotAtScheduleThread>(&self, _nast: NAST) -> std::cell::Ref<'_, Vec<LockAttempt>> {
         self.tx.1.0.borrow()
     }
 

From 3e9f9dee1a8c04dc3250e82e2ebd2b268cdaf26b Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 3 Sep 2022 10:07:44 +0900
Subject: [PATCH 1682/3199] save

---
 scheduler/src/lib.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 595a2b9a310526..58d583f357b923 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -486,9 +486,9 @@ impl LockAttemptsInCell {
 // commit_time  -+
 
 impl Task {
-    pub fn new_for_queue(unique_weight: UniqueWeight, tx: (SanitizedTransaction, Vec<LockAttempt>)) -> std::sync::Arc<Self> {
+    pub fn new_for_queue<NAST: NotAtScheduleThread>(nast: NAST, unique_weight: UniqueWeight, tx: (SanitizedTransaction, Vec<LockAttempt>)) -> std::sync::Arc<Self> {
         TaskInQueue::new(Self {
-            for_indexer: tx.1.iter().map(|a| a.clone_for_test()).collect(),
+            for_indexer: tx.1.iter().map(|a| a.clone_for_test(nast)).collect(),
             unique_weight,
             tx: (tx.0, LockAttemptsInCell::new(std::cell::RefCell::new(tx.1))),
             contention_count: 0,

From 479b57cec05e3c31c89e22c819411753ff4353fe Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 3 Sep 2022 10:08:36 +0900
Subject: [PATCH 1683/3199] save

---
 scheduler/src/lib.rs | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 58d583f357b923..1963fd745f5cdd 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -570,11 +570,11 @@ impl Task {
     }
 
 
-    pub fn clone_for_test(&self) -> Self {
+    pub fn clone_for_test<NAST: NotAtTopOfScheduleThread>(&self, nast: NAST) -> Self {
         Self {
             unique_weight: self.unique_weight,
-            for_indexer: self.lock_attempts_not_mut().iter().map(|a| a.clone_for_test()).collect(),
-            tx: (self.tx.0.clone(), LockAttemptsInCell::new(std::cell::RefCell::new(self.lock_attempts_not_mut().iter().map(|l| l.clone_for_test()).collect::<Vec<_>>()))),
+            for_indexer: self.lock_attempts_not_mut().iter().map(|a| a.clone_for_test(nast)).collect(),
+            tx: (self.tx.0.clone(), LockAttemptsInCell::new(std::cell::RefCell::new(self.lock_attempts_not_mut().iter().map(|l| l.clone_for_test(nast)).collect::<Vec<_>>()))),
             contention_count: Default::default(),
             uncontended: Default::default(),
             sequence_time: std::sync::atomic::AtomicUsize::new(usize::max_value()),

From 127f401f32a4762e82c04da887e8e27eaccb9e4c Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 3 Sep 2022 10:08:54 +0900
Subject: [PATCH 1684/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 1963fd745f5cdd..4deb893526e043 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -570,7 +570,7 @@ impl Task {
     }
 
 
-    pub fn clone_for_test<NAST: NotAtTopOfScheduleThread>(&self, nast: NAST) -> Self {
+    pub fn clone_for_test<NAST: NotAtScheduleThread>(&self, nast: NAST) -> Self {
         Self {
             unique_weight: self.unique_weight,
             for_indexer: self.lock_attempts_not_mut().iter().map(|a| a.clone_for_test(nast)).collect(),

From 0458bcb54995f2579a2a5ea0e874116f4c15686b Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 3 Sep 2022 10:09:14 +0900
Subject: [PATCH 1685/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 4deb893526e043..424e877277247b 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -574,7 +574,7 @@ impl Task {
         Self {
             unique_weight: self.unique_weight,
             for_indexer: self.lock_attempts_not_mut().iter().map(|a| a.clone_for_test(nast)).collect(),
-            tx: (self.tx.0.clone(), LockAttemptsInCell::new(std::cell::RefCell::new(self.lock_attempts_not_mut().iter().map(|l| l.clone_for_test(nast)).collect::<Vec<_>>()))),
+            tx: (self.tx.0.clone(), LockAttemptsInCell::new(std::cell::RefCell::new(self.lock_attempts_not_mut(nast).iter().map(|l| l.clone_for_test(nast)).collect::<Vec<_>>()))),
             contention_count: Default::default(),
             uncontended: Default::default(),
             sequence_time: std::sync::atomic::AtomicUsize::new(usize::max_value()),

From 55118549e01bbcd647065a324c92717778591391 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 3 Sep 2022 10:09:27 +0900
Subject: [PATCH 1686/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 424e877277247b..02dda3d4842fc3 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -574,7 +574,7 @@ impl Task {
         Self {
             unique_weight: self.unique_weight,
             for_indexer: self.lock_attempts_not_mut().iter().map(|a| a.clone_for_test(nast)).collect(),
-            tx: (self.tx.0.clone(), LockAttemptsInCell::new(std::cell::RefCell::new(self.lock_attempts_not_mut(nast).iter().map(|l| l.clone_for_test(nast)).collect::<Vec<_>>()))),
+            tx: (self.tx.0.clone(), LockAttemptsInCell::new(std::cell::RefCell::new(self.lock_attempts_not_mut(nast).iter().map(|l| l.clone_for_test()).collect::<Vec<_>>()))),
             contention_count: Default::default(),
             uncontended: Default::default(),
             sequence_time: std::sync::atomic::AtomicUsize::new(usize::max_value()),

From 4c9162591ad4d0f7bb78e14603cb2ed606a85aeb Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 3 Sep 2022 10:09:41 +0900
Subject: [PATCH 1687/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 02dda3d4842fc3..2a848a2c15fa69 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -573,7 +573,7 @@ impl Task {
     pub fn clone_for_test<NAST: NotAtScheduleThread>(&self, nast: NAST) -> Self {
         Self {
             unique_weight: self.unique_weight,
-            for_indexer: self.lock_attempts_not_mut().iter().map(|a| a.clone_for_test(nast)).collect(),
+            for_indexer: self.lock_attempts_not_mut().iter().map(|a| a.clone_for_test()).collect(),
             tx: (self.tx.0.clone(), LockAttemptsInCell::new(std::cell::RefCell::new(self.lock_attempts_not_mut(nast).iter().map(|l| l.clone_for_test()).collect::<Vec<_>>()))),
             contention_count: Default::default(),
             uncontended: Default::default(),

From 660309616f577ed109cc87c425a54b58a1b86ec3 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 3 Sep 2022 10:09:55 +0900
Subject: [PATCH 1688/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 2a848a2c15fa69..42ebbcd9b7c3ae 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -573,7 +573,7 @@ impl Task {
     pub fn clone_for_test<NAST: NotAtScheduleThread>(&self, nast: NAST) -> Self {
         Self {
             unique_weight: self.unique_weight,
-            for_indexer: self.lock_attempts_not_mut().iter().map(|a| a.clone_for_test()).collect(),
+            for_indexer: self.lock_attempts_not_mut(nast).iter().map(|a| a.clone_for_test()).collect(),
             tx: (self.tx.0.clone(), LockAttemptsInCell::new(std::cell::RefCell::new(self.lock_attempts_not_mut(nast).iter().map(|l| l.clone_for_test()).collect::<Vec<_>>()))),
             contention_count: Default::default(),
             uncontended: Default::default(),

From b1a6844c830b8b95baa2a5ea23c08810b00c8473 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 3 Sep 2022 10:10:16 +0900
Subject: [PATCH 1689/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 42ebbcd9b7c3ae..aa1063d795c92d 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -488,7 +488,7 @@ impl LockAttemptsInCell {
 impl Task {
     pub fn new_for_queue<NAST: NotAtScheduleThread>(nast: NAST, unique_weight: UniqueWeight, tx: (SanitizedTransaction, Vec<LockAttempt>)) -> std::sync::Arc<Self> {
         TaskInQueue::new(Self {
-            for_indexer: tx.1.iter().map(|a| a.clone_for_test(nast)).collect(),
+            for_indexer: tx.1.iter().map(|a| a.clone_for_test()).collect(),
             unique_weight,
             tx: (tx.0, LockAttemptsInCell::new(std::cell::RefCell::new(tx.1))),
             contention_count: 0,

From ac2e38e0b3d2179b59a22bfcf3c4836f8da51a81 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 3 Sep 2022 10:10:43 +0900
Subject: [PATCH 1690/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index aa1063d795c92d..57cbbc80021c54 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1068,7 +1068,7 @@ impl ScheduleStage {
                 #[derive(Clone, Copy, Debug)]
                 struct NotAtTopOfScheduleThread;
                 unsafe impl NotAtScheduleThread for NotAtTopOfScheduleThread {}
-                let nast = NotAtScheduleThread;
+                let nast = NotAtTopOfScheduleThread;
 
                 while let Ok(mut a) = ee_receiver.recv() {
                     assert!(a.task.lock_attempts_not_mut(nast).is_empty());

From 68ad54e8dca4cdedbfe0f387fa2fc79f3d518488 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 3 Sep 2022 10:11:17 +0900
Subject: [PATCH 1691/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 57cbbc80021c54..da8210c9333541 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -95,7 +95,7 @@ impl ExecutionEnvironment {
 }
 
 unsafe trait AtScheduleThread: Copy {}
-unsafe trait NotAtScheduleThread: Copy {}
+unsafe pub trait NotAtScheduleThread: Copy {}
 
 impl PageRc {
     fn page_mut<AST: AtScheduleThread>(&self, _ast: AST) -> std::cell::RefMut<'_, Page> {

From 33f56eed67a2fbd853dd4a0177b5f53abc570698 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 3 Sep 2022 10:11:24 +0900
Subject: [PATCH 1692/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index da8210c9333541..c7f843f5b70f27 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -95,7 +95,7 @@ impl ExecutionEnvironment {
 }
 
 unsafe trait AtScheduleThread: Copy {}
-unsafe pub trait NotAtScheduleThread: Copy {}
+pub unsafe trait NotAtScheduleThread: Copy {}
 
 impl PageRc {
     fn page_mut<AST: AtScheduleThread>(&self, _ast: AST) -> std::cell::RefMut<'_, Page> {

From cadbb14045bd2d1da518cb6f7d14356f330b20ce Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 3 Sep 2022 10:12:32 +0900
Subject: [PATCH 1693/3199] save

---
 transaction-scheduler-bench/src/main.rs | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/transaction-scheduler-bench/src/main.rs b/transaction-scheduler-bench/src/main.rs
index 61f814f7e0dff0..6695db83e3b583 100644
--- a/transaction-scheduler-bench/src/main.rs
+++ b/transaction-scheduler-bench/src/main.rs
@@ -476,6 +476,10 @@ fn send_packets(
             Ordering::Relaxed,
         );
         let mut rng = rand::thread_rng();
+        #[derive(Clone, Copy, Debug)]
+        struct NotAtTopOfScheduleThread;
+        unsafe impl NotAtScheduleThread for NotAtTopOfScheduleThread {}
+        let nast = NotAtTopOfScheduleThread;
 
         for vv in packet_batches {
             for v in vv {
@@ -483,7 +487,7 @@ fn send_packets(
                 //let p = (p << 32) | (rng.gen::<u64>() & 0x0000_0000_ffff_ffff);
                 let uw = unique_weight.fetch_sub(1, std::sync::atomic::Ordering::SeqCst);
                 let p = (p << 32) | (uw & 0x0000_0000_ffff_ffff);
-                let t = solana_scheduler::Task::new_for_queue(p, v);
+                let t = solana_scheduler::Task::new_for_queue(nast, p, v);
                 packet_batch_sender.send(t).unwrap();
             }
         }

From d36facde67e7d6d91b5aa78a0404b0d44323decf Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 3 Sep 2022 10:12:47 +0900
Subject: [PATCH 1694/3199] save

---
 transaction-scheduler-bench/src/main.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/transaction-scheduler-bench/src/main.rs b/transaction-scheduler-bench/src/main.rs
index 6695db83e3b583..f8c3bc64f729ed 100644
--- a/transaction-scheduler-bench/src/main.rs
+++ b/transaction-scheduler-bench/src/main.rs
@@ -478,7 +478,7 @@ fn send_packets(
         let mut rng = rand::thread_rng();
         #[derive(Clone, Copy, Debug)]
         struct NotAtTopOfScheduleThread;
-        unsafe impl NotAtScheduleThread for NotAtTopOfScheduleThread {}
+        unsafe impl solana_scheduler::NotAtScheduleThread for NotAtTopOfScheduleThread {}
         let nast = NotAtTopOfScheduleThread;
 
         for vv in packet_batches {

From 5b92bccb5df3e93d5be54c0a62b1f633b34cc568 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 3 Sep 2022 10:23:07 +0900
Subject: [PATCH 1695/3199] save

---
 scheduler/src/lib.rs | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index c7f843f5b70f27..d7c85e064ca99d 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -457,7 +457,7 @@ struct Bundle {
 pub struct Task {
     unique_weight: UniqueWeight,
     pub tx: (SanitizedTransaction, LockAttemptsInCell), // actually should be Bundle
-    pub contention_count: usize,
+    pub contention_count: std::sync::atomic::AtomicUsize,
     pub uncontended: std::sync::atomic::AtomicUsize,
     pub sequence_time: std::sync::atomic::AtomicUsize,
     pub sequence_end_time: std::sync::atomic::AtomicUsize,
@@ -491,7 +491,7 @@ impl Task {
             for_indexer: tx.1.iter().map(|a| a.clone_for_test()).collect(),
             unique_weight,
             tx: (tx.0, LockAttemptsInCell::new(std::cell::RefCell::new(tx.1))),
-            contention_count: 0,
+            contention_count: Default::default(),
             uncontended: Default::default(),
             sequence_time: std::sync::atomic::AtomicUsize::new(usize::max_value()),
             sequence_end_time: std::sync::atomic::AtomicUsize::new(usize::max_value()),
@@ -811,7 +811,7 @@ impl ScheduleStage {
                     from_runnable,
                 );
                 let lock_count = next_task.lock_attempts_mut(ast).len();
-                panic!();//next_task.contention_count += 1;
+                next_task.contention_count.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
 
                 if from_runnable {
                     trace!("move to contended due to lock failure [{}/{}/{}]", unlockable_count, provisional_count, lock_count);

From 9166732cbf47b126406f599086f37927e6ddb1ff Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 3 Sep 2022 10:23:29 +0900
Subject: [PATCH 1696/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index d7c85e064ca99d..f830794df58307 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -54,7 +54,7 @@ impl ExecutionEnvironment {
     pub fn reindex(&mut self) {
         let uq = self.unique_weight;
         //self.task.trace_timestamps("in_exec(self)");
-        let should_remove = self.task.contention_count > 0;
+        let should_remove = self.task.contention_count.load(std::sync::atomic::Ordering::SeqCst) > 0;
         for mut lock_attempt in self.finalized_lock_attempts.iter_mut() {
             let contended_unique_weights = lock_attempt.contended_unique_weights();
             contended_unique_weights.heaviest_task_cursor().map(|mut task_cursor| {

From 1932b08b734722e5e748ae06f0087482b1c9102c Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 3 Sep 2022 10:23:44 +0900
Subject: [PATCH 1697/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index f830794df58307..c831729e66b91a 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -853,7 +853,7 @@ impl ScheduleStage {
                 continue;
             }
 
-            trace!("successful lock: (from_runnable: {}) after {} contentions", from_runnable, next_task.contention_count);
+            trace!("successful lock: (from_runnable: {}) after {} contentions", from_runnable, next_task.contention_count.load(std::sync::atomic::Ordering::SeqCst));
 
             if !from_runnable {
                 *contended_count = contended_count.checked_sub(1).unwrap();

From 14bb244021cd55a95444204623b8f0bc749651cb Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 3 Sep 2022 10:23:59 +0900
Subject: [PATCH 1698/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index c831729e66b91a..71a2c697dcddfc 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -824,7 +824,7 @@ impl ScheduleStage {
 
                     // maybe run lightweight prune logic on contended_queue here.
                 } else {
-                    trace!("relock failed [{}/{}/{}]; remains in contended: {:?} contention: {}", unlockable_count, provisional_count, lock_count, &unique_weight, next_task.contention_count);
+                    trace!("relock failed [{}/{}/{}]; remains in contended: {:?} contention: {}", unlockable_count, provisional_count, lock_count, &unique_weight, next_task.contention_count.load(std::sync::atomic::Ordering::SeqCst));
                     //address_book.uncontended_task_ids.clear();
                 }
 

From 9d64210acdaf7f1c740b23d2951bd6d06879f353 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 3 Sep 2022 10:25:19 +0900
Subject: [PATCH 1699/3199] save

---
 scheduler/src/lib.rs | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 71a2c697dcddfc..8c9b506cbdb5b5 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -949,8 +949,6 @@ impl ScheduleStage {
             if page.current_usage == Usage::Unused && page.next_usage != Usage::Unused {
                 page.switch_to_next_usage();
                 for mut tracker in std::mem::take(&mut page.provisional_task_ids).into_iter() {
-                    panic!();
-                    /*
                     let tracker = unsafe { std::sync::Arc::<ProvisioningTracker>::get_mut_unchecked(&mut tracker) };
                     tracker.progress();
                     if tracker.is_fulfilled() {
@@ -960,7 +958,6 @@ impl ScheduleStage {
                     } else {
                         trace!("provisioning tracker progress: {} => {}", tracker.prev_count(), tracker.count());
                     }
-                    */
                 }
             }
 

From 32ad128ea40fd83dc58ff9533c47966f142f6b73 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 3 Sep 2022 10:25:34 +0900
Subject: [PATCH 1700/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 8c9b506cbdb5b5..f17a08f3a08e70 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1,5 +1,5 @@
 #![feature(map_first_last)]
-//#![feature(get_mut_unchecked)]
+#![feature(get_mut_unchecked)]
 
 use {
     crossbeam_channel::{bounded, unbounded},

From 5288c510d294b89b1db1811dd8fe66e470f1b47e Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 3 Sep 2022 10:26:10 +0900
Subject: [PATCH 1701/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index f17a08f3a08e70..68d9cb65d30f19 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -820,7 +820,7 @@ impl ScheduleStage {
                     //for lock_attempt in next_task.tx.1.iter() {
                     //    lock_attempt.contended_unique_weights().insert_task(unique_weight, TaskInQueue::clone(&a2));
                     //}
-                    panic!(); // task_sender.send((TaskInQueue::clone(&next_task), std::mem::take(&mut next_task.for_indexer))).unwrap();
+                    task_sender.send((TaskInQueue::clone(&next_task), std::mem::take(&mut next_task.for_indexer))).unwrap();
 
                     // maybe run lightweight prune logic on contended_queue here.
                 } else {

From 5b23f958272845f12ff20b573482e2b0bf3cfb02 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 3 Sep 2022 10:27:43 +0900
Subject: [PATCH 1702/3199] save

---
 scheduler/src/lib.rs | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 68d9cb65d30f19..1a2fe97095a11f 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -820,7 +820,8 @@ impl ScheduleStage {
                     //for lock_attempt in next_task.tx.1.iter() {
                     //    lock_attempt.contended_unique_weights().insert_task(unique_weight, TaskInQueue::clone(&a2));
                     //}
-                    task_sender.send((TaskInQueue::clone(&next_task), std::mem::take(&mut next_task.for_indexer))).unwrap();
+                    let nn = unsafe { TaskInQueue::get_mut_unchecked(&mut next_task) };
+                    task_sender.send((TaskInQueue::clone(&next_task), std::mem::take(&mut nn.for_indexer))).unwrap();
 
                     // maybe run lightweight prune logic on contended_queue here.
                 } else {

From da80968a5a39cc8b75840d7d28e7507a7f280071 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 3 Sep 2022 10:28:04 +0900
Subject: [PATCH 1703/3199] save

---
 scheduler/src/lib.rs | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 1a2fe97095a11f..5b1fba73b12dd5 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -820,8 +820,9 @@ impl ScheduleStage {
                     //for lock_attempt in next_task.tx.1.iter() {
                     //    lock_attempt.contended_unique_weights().insert_task(unique_weight, TaskInQueue::clone(&a2));
                     //}
+                    let a = TaskInQueue::clone(&next_task);
                     let nn = unsafe { TaskInQueue::get_mut_unchecked(&mut next_task) };
-                    task_sender.send((TaskInQueue::clone(&next_task), std::mem::take(&mut nn.for_indexer))).unwrap();
+                    task_sender.send((a, std::mem::take(&mut nn.for_indexer))).unwrap();
 
                     // maybe run lightweight prune logic on contended_queue here.
                 } else {

From b9658fdc7da6259ed4e373da543423604632b1fb Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 3 Sep 2022 10:28:57 +0900
Subject: [PATCH 1704/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 5b1fba73b12dd5..35cdda8e5b33e8 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -821,7 +821,7 @@ impl ScheduleStage {
                     //    lock_attempt.contended_unique_weights().insert_task(unique_weight, TaskInQueue::clone(&a2));
                     //}
                     let a = TaskInQueue::clone(&next_task);
-                    let nn = unsafe { TaskInQueue::get_mut_unchecked(&mut next_task) };
+                    let nn = unsafe { TaskInQueue::get_mut_unchecked(next_task) };
                     task_sender.send((a, std::mem::take(&mut nn.for_indexer))).unwrap();
 
                     // maybe run lightweight prune logic on contended_queue here.

From fa5069fd5c03384013357f57fe5a80d136dc2d1e Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 3 Sep 2022 10:29:14 +0900
Subject: [PATCH 1705/3199] save

---
 scheduler/src/lib.rs | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 35cdda8e5b33e8..95eb5b0d5ddcfd 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -821,7 +821,8 @@ impl ScheduleStage {
                     //    lock_attempt.contended_unique_weights().insert_task(unique_weight, TaskInQueue::clone(&a2));
                     //}
                     let a = TaskInQueue::clone(&next_task);
-                    let nn = unsafe { TaskInQueue::get_mut_unchecked(next_task) };
+                    let tt = &mut next_task;
+                    let nn = unsafe { TaskInQueue::get_mut_unchecked(tt) };
                     task_sender.send((a, std::mem::take(&mut nn.for_indexer))).unwrap();
 
                     // maybe run lightweight prune logic on contended_queue here.

From 2b4175a682db70e23eb3d20f78189b846634cc96 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 3 Sep 2022 10:32:08 +0900
Subject: [PATCH 1706/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 95eb5b0d5ddcfd..8daefdf9ff7fce 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -465,7 +465,7 @@ pub struct Task {
     pub queue_end_time: std::sync::atomic::AtomicUsize,
     pub execute_time: std::sync::atomic::AtomicUsize,
     pub commit_time: std::sync::atomic::AtomicUsize,
-    pub for_indexer: Vec<LockAttempt>,
+    pub for_indexer: LockAttemptsInCell,
 }
 
 #[derive(Debug)]

From 7bd156add920d273096cabab1a18faa37450b51a Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 3 Sep 2022 10:33:02 +0900
Subject: [PATCH 1707/3199] save

---
 scheduler/src/lib.rs | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 8daefdf9ff7fce..b9b19d38c7fe74 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -821,9 +821,7 @@ impl ScheduleStage {
                     //    lock_attempt.contended_unique_weights().insert_task(unique_weight, TaskInQueue::clone(&a2));
                     //}
                     let a = TaskInQueue::clone(&next_task);
-                    let tt = &mut next_task;
-                    let nn = unsafe { TaskInQueue::get_mut_unchecked(tt) };
-                    task_sender.send((a, std::mem::take(&mut nn.for_indexer))).unwrap();
+                    task_sender.send((a, std::mem::take(&mut *nn.for_indexer.borrow_mut()))).unwrap();
 
                     // maybe run lightweight prune logic on contended_queue here.
                 } else {

From 63d96f9749b94f644f898d9c86bd3baedc91e560 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 3 Sep 2022 10:33:20 +0900
Subject: [PATCH 1708/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index b9b19d38c7fe74..aae44fa8d2fbbd 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -573,7 +573,7 @@ impl Task {
     pub fn clone_for_test<NAST: NotAtScheduleThread>(&self, nast: NAST) -> Self {
         Self {
             unique_weight: self.unique_weight,
-            for_indexer: self.lock_attempts_not_mut(nast).iter().map(|a| a.clone_for_test()).collect(),
+            for_indexer: LockAttemptsInCell::new(self.lock_attempts_not_mut(nast).iter().map(|a| a.clone_for_test()).collect()),
             tx: (self.tx.0.clone(), LockAttemptsInCell::new(std::cell::RefCell::new(self.lock_attempts_not_mut(nast).iter().map(|l| l.clone_for_test()).collect::<Vec<_>>()))),
             contention_count: Default::default(),
             uncontended: Default::default(),

From 8293c7d0c015e114fc40b23c76c4888cf44ccf6e Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 3 Sep 2022 10:33:46 +0900
Subject: [PATCH 1709/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index aae44fa8d2fbbd..49d03f30abc777 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -573,7 +573,7 @@ impl Task {
     pub fn clone_for_test<NAST: NotAtScheduleThread>(&self, nast: NAST) -> Self {
         Self {
             unique_weight: self.unique_weight,
-            for_indexer: LockAttemptsInCell::new(self.lock_attempts_not_mut(nast).iter().map(|a| a.clone_for_test()).collect()),
+            for_indexer: LockAttemptsInCell::new(self.lock_attempts_not_mut(nast).iter().map(|a| a.clone_for_test()).collect::<Vec<_>>()),
             tx: (self.tx.0.clone(), LockAttemptsInCell::new(std::cell::RefCell::new(self.lock_attempts_not_mut(nast).iter().map(|l| l.clone_for_test()).collect::<Vec<_>>()))),
             contention_count: Default::default(),
             uncontended: Default::default(),

From a52bfb9bb7d1d96f04e2ab3fbb363b6d0543f141 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 3 Sep 2022 10:34:15 +0900
Subject: [PATCH 1710/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 49d03f30abc777..738a3784d98eab 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -573,7 +573,7 @@ impl Task {
     pub fn clone_for_test<NAST: NotAtScheduleThread>(&self, nast: NAST) -> Self {
         Self {
             unique_weight: self.unique_weight,
-            for_indexer: LockAttemptsInCell::new(self.lock_attempts_not_mut(nast).iter().map(|a| a.clone_for_test()).collect::<Vec<_>>()),
+            for_indexer: LockAttemptsInCell::new(std::cell::RefCell::new(self.lock_attempts_not_mut(nast).iter().map(|a| a.clone_for_test()).collect())),
             tx: (self.tx.0.clone(), LockAttemptsInCell::new(std::cell::RefCell::new(self.lock_attempts_not_mut(nast).iter().map(|l| l.clone_for_test()).collect::<Vec<_>>()))),
             contention_count: Default::default(),
             uncontended: Default::default(),

From ff49c9a85759cc208396ceb3545e5c9d65f704ff Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 3 Sep 2022 10:34:35 +0900
Subject: [PATCH 1711/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 738a3784d98eab..a6a53d3226b565 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -488,7 +488,7 @@ impl LockAttemptsInCell {
 impl Task {
     pub fn new_for_queue<NAST: NotAtScheduleThread>(nast: NAST, unique_weight: UniqueWeight, tx: (SanitizedTransaction, Vec<LockAttempt>)) -> std::sync::Arc<Self> {
         TaskInQueue::new(Self {
-            for_indexer: tx.1.iter().map(|a| a.clone_for_test()).collect(),
+            for_indexer: LockAttemptsInCell::new(std::cell::RefCell::new(tx.1.iter().map(|a| a.clone_for_test()).collect())),
             unique_weight,
             tx: (tx.0, LockAttemptsInCell::new(std::cell::RefCell::new(tx.1))),
             contention_count: Default::default(),

From 21ebbbf1489158a9958c2529e96e2f35992372e5 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 3 Sep 2022 10:34:53 +0900
Subject: [PATCH 1712/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index a6a53d3226b565..58741cb853c63f 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -821,7 +821,7 @@ impl ScheduleStage {
                     //    lock_attempt.contended_unique_weights().insert_task(unique_weight, TaskInQueue::clone(&a2));
                     //}
                     let a = TaskInQueue::clone(&next_task);
-                    task_sender.send((a, std::mem::take(&mut *nn.for_indexer.borrow_mut()))).unwrap();
+                    task_sender.send((a, std::mem::take(&mut *next_task.for_indexer.borrow_mut()))).unwrap();
 
                     // maybe run lightweight prune logic on contended_queue here.
                 } else {

From ec492300143719f98a723b34648091fdf23343dd Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 3 Sep 2022 10:35:09 +0900
Subject: [PATCH 1713/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 58741cb853c63f..c3846936ee9088 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -821,7 +821,7 @@ impl ScheduleStage {
                     //    lock_attempt.contended_unique_weights().insert_task(unique_weight, TaskInQueue::clone(&a2));
                     //}
                     let a = TaskInQueue::clone(&next_task);
-                    task_sender.send((a, std::mem::take(&mut *next_task.for_indexer.borrow_mut()))).unwrap();
+                    task_sender.send((a, std::mem::take(&mut *next_task.for_indexer.0.borrow_mut()))).unwrap();
 
                     // maybe run lightweight prune logic on contended_queue here.
                 } else {

From faea1bb5242767e3ef393929c50a84a04f283b49 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 3 Sep 2022 10:58:21 +0900
Subject: [PATCH 1714/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index c3846936ee9088..1c78eda6e9ab4d 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -260,7 +260,7 @@ impl ProvisioningTracker {
         self.remaining_count == 0
     }
 
-    fn progress(&mut self) {
+    fn progress(&self) {
         self.remaining_count = self.remaining_count.checked_sub(1).unwrap();
     }
 

From b2b58c72d69227ec2689f2cda919a613deea7780 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 3 Sep 2022 10:58:59 +0900
Subject: [PATCH 1715/3199] save

---
 scheduler/src/lib.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 1c78eda6e9ab4d..7ffd734acbc655 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -247,13 +247,13 @@ pub struct AddressBook {
 
 #[derive(Debug)]
 struct ProvisioningTracker {
-    remaining_count: usize,
+    remaining_count: std::sync::atomic::AtomicUsize,
     task: TaskInQueue,
 }
 
 impl ProvisioningTracker {
     fn new(remaining_count: usize, task: TaskInQueue) -> Self {
-        Self { remaining_count, task }
+        Self { remaining_count: std::sync::atomic::AtomicUsize::new(remaining_count), task }
     }
 
     fn is_fulfilled(&self) -> bool {

From a13b25c34785693263ec947e424590c3f8fa7526 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 3 Sep 2022 11:00:03 +0900
Subject: [PATCH 1716/3199] save

---
 scheduler/src/lib.rs | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 7ffd734acbc655..f61879723eb3cd 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -261,15 +261,15 @@ impl ProvisioningTracker {
     }
 
     fn progress(&self) {
-        self.remaining_count = self.remaining_count.checked_sub(1).unwrap();
+        self.remaining_count.fetch_sub(1, std::sync::atomic::Ordering::SeqCst);
     }
 
     fn prev_count(&self) -> usize {
-        self.remaining_count + 1
+        self.count() + 1
     }
 
     fn count(&self) -> usize {
-        self.remaining_count
+        self.remaining_count.load(std::sync::atomic::Ordering::SeqCst)
     }
 }
 

From b461d7350a335660bda211ccbf158b10f6d1bada Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 3 Sep 2022 11:00:13 +0900
Subject: [PATCH 1717/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index f61879723eb3cd..8a57e67920eaea 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -257,7 +257,7 @@ impl ProvisioningTracker {
     }
 
     fn is_fulfilled(&self) -> bool {
-        self.remaining_count == 0
+        self.count() == 0
     }
 
     fn progress(&self) {

From dcfa49898789e371ce122c53a65cce012cb523ba Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 3 Sep 2022 11:00:36 +0900
Subject: [PATCH 1718/3199] save

---
 scheduler/src/lib.rs | 1 -
 1 file changed, 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 8a57e67920eaea..7f71c8690c94e6 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -950,7 +950,6 @@ impl ScheduleStage {
             if page.current_usage == Usage::Unused && page.next_usage != Usage::Unused {
                 page.switch_to_next_usage();
                 for mut tracker in std::mem::take(&mut page.provisional_task_ids).into_iter() {
-                    let tracker = unsafe { std::sync::Arc::<ProvisioningTracker>::get_mut_unchecked(&mut tracker) };
                     tracker.progress();
                     if tracker.is_fulfilled() {
                         trace!("provisioning tracker progress: {} => {} (!)", tracker.prev_count(), tracker.count());

From 8fb1823212f72fc7d3abaddafdd33b3aa17d5928 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 3 Sep 2022 11:00:43 +0900
Subject: [PATCH 1719/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 7f71c8690c94e6..5dd9d1d5e6ca25 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -949,7 +949,7 @@ impl ScheduleStage {
             }
             if page.current_usage == Usage::Unused && page.next_usage != Usage::Unused {
                 page.switch_to_next_usage();
-                for mut tracker in std::mem::take(&mut page.provisional_task_ids).into_iter() {
+                for tracker in std::mem::take(&mut page.provisional_task_ids).into_iter() {
                     tracker.progress();
                     if tracker.is_fulfilled() {
                         trace!("provisioning tracker progress: {} => {} (!)", tracker.prev_count(), tracker.count());

From 78b01853a4d64ee8a41bb079cdcb071be890074c Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 3 Sep 2022 11:00:55 +0900
Subject: [PATCH 1720/3199] save

---
 scheduler/src/lib.rs | 1 -
 1 file changed, 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 5dd9d1d5e6ca25..865683f8f7ab3b 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1,5 +1,4 @@
 #![feature(map_first_last)]
-#![feature(get_mut_unchecked)]
 
 use {
     crossbeam_channel::{bounded, unbounded},

From 8c61d02ff05377f39f34b90115a452388d0b68fb Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 3 Sep 2022 11:02:41 +0900
Subject: [PATCH 1721/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 865683f8f7ab3b..656fff6ad85216 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -485,7 +485,7 @@ impl LockAttemptsInCell {
 // commit_time  -+
 
 impl Task {
-    pub fn new_for_queue<NAST: NotAtScheduleThread>(nast: NAST, unique_weight: UniqueWeight, tx: (SanitizedTransaction, Vec<LockAttempt>)) -> std::sync::Arc<Self> {
+    pub fn new_for_queue<NAST: NotAtScheduleThread>(nast: NAST, unique_weight: UniqueWeight, tx: (SanitizedTransaction, Vec<LockAttempt>)) -> TaskInQueue {
         TaskInQueue::new(Self {
             for_indexer: LockAttemptsInCell::new(std::cell::RefCell::new(tx.1.iter().map(|a| a.clone_for_test()).collect())),
             unique_weight,

From 6e90f93fdc8c5636244543011e3fdf28efb77078 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 3 Sep 2022 11:03:43 +0900
Subject: [PATCH 1722/3199] save

---
 scheduler/src/lib.rs | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 656fff6ad85216..c315b529f739f2 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -16,14 +16,14 @@ use {
 };
 
 /*
-type MyRcInner<T> = std::rc::Rc<T>;
+type PageRcInner<T> = std::rc::Rc<T>;
 unsafe impl Send for PageRc {}
 */
 
-type MyRcInner = std::sync::Arc<(std::cell::RefCell<Page>, TaskIds)>;
+type PageRcInner = std::sync::Arc<(std::cell::RefCell<Page>, TaskIds)>;
 
 #[derive(Debug, Clone)]
-pub struct PageRc(MyRcInner);
+pub struct PageRc(PageRcInner);
 unsafe impl Send for PageRc {}
 unsafe impl Sync for PageRc {}
 
@@ -422,7 +422,7 @@ pub struct Preloader {
 impl Preloader {
     #[inline(never)]
     pub fn load(&self, address: Pubkey) -> PageRc {
-        PageRc::clone(&self.book.entry(address).or_insert_with(|| PageRc(MyRcInner::new((core::cell::RefCell::new(Page::new(Usage::unused())), Default::default())))))
+        PageRc::clone(&self.book.entry(address).or_insert_with(|| PageRc(PageRcInner::new((core::cell::RefCell::new(Page::new(Usage::unused())), Default::default())))))
     }
 }
 

From 3851a4f67a702bfd0e695c87a86b8fd74abd7b08 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 3 Sep 2022 11:16:14 +0900
Subject: [PATCH 1723/3199] save

---
 scheduler/src/lib.rs | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index c315b529f739f2..0652869cd848eb 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -618,6 +618,11 @@ pub struct TaskQueue {
 }
 
 pub type TaskInQueue = std::sync::Arc<Task>;
+
+impl TaskInQueue {
+    fn clone2() {}
+}
+
 //type TaskQueueEntry<'a> = im::ordmap::Entry<'a, UniqueWeight, TaskInQueue>;
 //type TaskQueueOccupiedEntry<'a> = im::ordmap::OccupiedEntry<'a, UniqueWeight, TaskInQueue>;
 //type TaskQueueEntry<'a> = im::hashmap::Entry<'a, UniqueWeight, TaskInQueue, std::collections::hash_map::RandomState>;

From 373074b60eafc734f03e9d42cd7fcace221b6cf3 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 3 Sep 2022 11:17:57 +0900
Subject: [PATCH 1724/3199] save

---
 scheduler/src/lib.rs | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 0652869cd848eb..34b25157d4166a 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -501,6 +501,10 @@ impl Task {
         })
     }
 
+    pub fn clone_in_queue(this: &TaskInQueue) -> TaskInQueue {
+        TaskInQueue::clone(this)
+    }
+
     fn lock_attempts_mut<AST: AtScheduleThread>(&self, _ast: AST) -> std::cell::RefMut<'_, Vec<LockAttempt>> {
         self.tx.1.0.borrow_mut()
     }
@@ -619,10 +623,6 @@ pub struct TaskQueue {
 
 pub type TaskInQueue = std::sync::Arc<Task>;
 
-impl TaskInQueue {
-    fn clone2() {}
-}
-
 //type TaskQueueEntry<'a> = im::ordmap::Entry<'a, UniqueWeight, TaskInQueue>;
 //type TaskQueueOccupiedEntry<'a> = im::ordmap::OccupiedEntry<'a, UniqueWeight, TaskInQueue>;
 //type TaskQueueEntry<'a> = im::hashmap::Entry<'a, UniqueWeight, TaskInQueue, std::collections::hash_map::RandomState>;

From 47b4cd815077859705276c844bc526385f839d04 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 3 Sep 2022 11:18:30 +0900
Subject: [PATCH 1725/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 34b25157d4166a..05e16bb2a6e30a 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -824,7 +824,7 @@ impl ScheduleStage {
                     //for lock_attempt in next_task.tx.1.iter() {
                     //    lock_attempt.contended_unique_weights().insert_task(unique_weight, TaskInQueue::clone(&a2));
                     //}
-                    let a = TaskInQueue::clone(&next_task);
+                    let a = Task::clone_in_queue(&next_task);
                     task_sender.send((a, std::mem::take(&mut *next_task.for_indexer.0.borrow_mut()))).unwrap();
 
                     // maybe run lightweight prune logic on contended_queue here.

From ec9f7ebe6e82f036dfcb59af5e8923c4b79d761a Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 3 Sep 2022 11:19:32 +0900
Subject: [PATCH 1726/3199] save

---
 scheduler/src/lib.rs | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 05e16bb2a6e30a..f58dc6828b83f4 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -83,7 +83,7 @@ impl ExecutionEnvironment {
                 if should_remove && !removed {
                     contended_unique_weights.remove_task(&uq);
                 }
-                found.then(|| TaskInQueue::clone(task))
+                found.then(|| Task::clone_in_queue(task))
             }).flatten().map(|task| {
                 //task.trace_timestamps(&format!("in_exec(heaviest:{})", self.task.queue_time_label()));
                 lock_attempt.heaviest_uncontended = Some(task);
@@ -822,7 +822,7 @@ impl ScheduleStage {
                     next_task.mark_as_contended();
                     *contended_count = contended_count.checked_add(1).unwrap();
                     //for lock_attempt in next_task.tx.1.iter() {
-                    //    lock_attempt.contended_unique_weights().insert_task(unique_weight, TaskInQueue::clone(&a2));
+                    //    lock_attempt.contended_unique_weights().insert_task(unique_weight, Task::clone_in_queue(&a2));
                     //}
                     let a = Task::clone_in_queue(&next_task);
                     task_sender.send((a, std::mem::take(&mut *next_task.for_indexer.0.borrow_mut()))).unwrap();
@@ -845,7 +845,7 @@ impl ScheduleStage {
                 trace!("provisional exec: [{}/{}]", provisional_count, lock_count);
                 *contended_count = contended_count.checked_sub(1).unwrap();
                 next_task.mark_as_uncontended();
-                let tracker = std::sync::Arc::new(ProvisioningTracker::new(provisional_count, TaskInQueue::clone(&next_task)));
+                let tracker = std::sync::Arc::new(ProvisioningTracker::new(provisional_count, Task::clone_in_queue(&next_task)));
                 *provisioning_tracker_count = provisioning_tracker_count.checked_add(1).unwrap();
                 Self::finalize_lock_for_provisional_execution(
                     ast,
@@ -943,7 +943,7 @@ impl ScheduleStage {
                                     break;
                                 }
                             }
-                            found.then(|| TaskInQueue::clone(task))
+                            found.then(|| Task::clone_in_queue(task))
                         }).flatten().map(|task| {
                             address_book.uncontended_task_ids.insert(task.unique_weight, task);
                             ()
@@ -957,7 +957,7 @@ impl ScheduleStage {
                     tracker.progress();
                     if tracker.is_fulfilled() {
                         trace!("provisioning tracker progress: {} => {} (!)", tracker.prev_count(), tracker.count());
-                        address_book.fulfilled_provisional_task_ids.insert(tracker.task.unique_weight, TaskInQueue::clone(&tracker.task));
+                        address_book.fulfilled_provisional_task_ids.insert(tracker.task.unique_weight, Task::clone_in_queue(&tracker.task));
                         *provisioning_tracker_count = provisioning_tracker_count.checked_sub(1).unwrap();
                     } else {
                         trace!("provisioning tracker progress: {} => {}", tracker.prev_count(), tracker.count());
@@ -1096,7 +1096,7 @@ impl ScheduleStage {
                         if task.already_finished() {
                             break;
                         }
-                        lock_attempt.contended_unique_weights().insert_task(task.unique_weight, TaskInQueue::clone(&task));
+                        lock_attempt.contended_unique_weights().insert_task(task.unique_weight, Task::clone_in_queue(&task));
                     }
                 }
                 assert_eq!(task_receiver.len(), 0);

From 593940bd388064b6638af4174c9425fa8589cb55 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 3 Sep 2022 11:19:48 +0900
Subject: [PATCH 1727/3199] save

---
 scheduler/src/lib.rs | 1 +
 1 file changed, 1 insertion(+)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index f58dc6828b83f4..425572c12e49aa 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -501,6 +501,7 @@ impl Task {
         })
     }
 
+    #[inline(never)]
     pub fn clone_in_queue(this: &TaskInQueue) -> TaskInQueue {
         TaskInQueue::clone(this)
     }

From ab36315963bb66bb31e522ede87dfe9ae83f20db Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 3 Sep 2022 11:27:24 +0900
Subject: [PATCH 1728/3199] save

---
 scheduler/Cargo.toml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/scheduler/Cargo.toml b/scheduler/Cargo.toml
index 893de70089827e..710c04710af694 100644
--- a/scheduler/Cargo.toml
+++ b/scheduler/Cargo.toml
@@ -24,6 +24,7 @@ solana-version = { path = "../version", version = "=1.11.5" }
 solana-metrics = { path = "../metrics", version = "=1.11.5" }
 solana-measure = { path = "../measure", version = "=1.11.5" }
 sha2 = { version = "0.10.2" }
+triomphe = "0.1.7"
 rand = "0.7.0"
 
 [[bin]]

From ce21715a0edd5264ca2362baa8d115ad25e9a0a8 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 3 Sep 2022 11:28:09 +0900
Subject: [PATCH 1729/3199] save

---
 Cargo.lock           | 11 +++++++++++
 scheduler/src/lib.rs |  2 +-
 2 files changed, 12 insertions(+), 1 deletion(-)

diff --git a/Cargo.lock b/Cargo.lock
index 0c82122a5b7e51..b1587a32e4069b 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -6099,6 +6099,7 @@ dependencies = [
  "solana-runtime",
  "solana-sdk 1.11.5",
  "solana-version",
+ "triomphe",
 ]
 
 [[package]]
@@ -7545,6 +7546,16 @@ version = "0.4.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "0de5f738ceab88e2491a94ddc33c3feeadfa95fedc60363ef110845df12f3878"
 
+[[package]]
+name = "triomphe"
+version = "0.1.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7fe1b3800b35f9b936c28dc59dbda91b195371269396784d931fe2a5a2be3d2f"
+dependencies = [
+ "serde",
+ "stable_deref_trait",
+]
+
 [[package]]
 name = "try-lock"
 version = "0.2.3"
diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 425572c12e49aa..ff189fcf62629f 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -622,7 +622,7 @@ pub struct TaskQueue {
     //tasks: std::sync::Arc<dashmap::DashMap<UniqueWeight, TaskInQueue>>,
 }
 
-pub type TaskInQueue = std::sync::Arc<Task>;
+pub type TaskInQueue = triomphe::Arc<Task>;
 
 //type TaskQueueEntry<'a> = im::ordmap::Entry<'a, UniqueWeight, TaskInQueue>;
 //type TaskQueueOccupiedEntry<'a> = im::ordmap::OccupiedEntry<'a, UniqueWeight, TaskInQueue>;

From 6bcdf23bfa49bbff33dce543372b14dba8112bbd Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 3 Sep 2022 11:28:40 +0900
Subject: [PATCH 1730/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index ff189fcf62629f..4def0133c1d657 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -846,7 +846,7 @@ impl ScheduleStage {
                 trace!("provisional exec: [{}/{}]", provisional_count, lock_count);
                 *contended_count = contended_count.checked_sub(1).unwrap();
                 next_task.mark_as_uncontended();
-                let tracker = std::sync::Arc::new(ProvisioningTracker::new(provisional_count, Task::clone_in_queue(&next_task)));
+                let tracker = triomphe::Arc::new(ProvisioningTracker::new(provisional_count, Task::clone_in_queue(&next_task)));
                 *provisioning_tracker_count = provisioning_tracker_count.checked_add(1).unwrap();
                 Self::finalize_lock_for_provisional_execution(
                     ast,

From c53f63160a893d9d3048dd8dd2efaa00b5fe513e Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 3 Sep 2022 11:29:04 +0900
Subject: [PATCH 1731/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 4def0133c1d657..278aaefdd4cc65 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -881,7 +881,7 @@ impl ScheduleStage {
         ast: AST,
         address_book: &mut AddressBook,
         next_task: &Task,
-        tracker: std::sync::Arc<ProvisioningTracker>,
+        tracker: triomphe::Arc<ProvisioningTracker>,
     ) {
         for l in next_task.lock_attempts_mut(ast).iter_mut() {
             match l.status {

From b815e3ea2d80a6db7eb9bec0616ffb04a3a32ee6 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 3 Sep 2022 11:29:14 +0900
Subject: [PATCH 1732/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 278aaefdd4cc65..ebec5f1203507f 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -886,7 +886,7 @@ impl ScheduleStage {
         for l in next_task.lock_attempts_mut(ast).iter_mut() {
             match l.status {
                 LockStatus::Provisional => {
-                    l.target.page_mut(ast).provisional_task_ids.push(std::sync::Arc::clone(&tracker));
+                    l.target.page_mut(ast).provisional_task_ids.push(triomphe::Arc::clone(&tracker));
                 }
                 LockStatus::Succeded => {
                     // do nothing

From 623942d8eabbd2de6059cd04b81914bad8737ada Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 3 Sep 2022 11:29:28 +0900
Subject: [PATCH 1733/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index ebec5f1203507f..a58a30c71d5f20 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -208,7 +208,7 @@ impl TaskIds {
 pub struct Page {
     current_usage: Usage,
     next_usage: Usage,
-    provisional_task_ids: Vec<std::sync::Arc<ProvisioningTracker>>,
+    provisional_task_ids: Vec<triomphe::Arc<ProvisioningTracker>>,
     //loaded account from Accounts db
     //comulative_cu for qos; i.e. track serialized cumulative keyed by addresses and bail out block
     //producing as soon as any one of cu from the executing thread reaches to the limit

From 9d520f217a94529b874c2f1e322e672412720a02 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 3 Sep 2022 11:29:56 +0900
Subject: [PATCH 1734/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index a58a30c71d5f20..ebbecfc5168258 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -20,7 +20,7 @@ type PageRcInner<T> = std::rc::Rc<T>;
 unsafe impl Send for PageRc {}
 */
 
-type PageRcInner = std::sync::Arc<(std::cell::RefCell<Page>, TaskIds)>;
+type PageRcInner = triomphe::Arc<(std::cell::RefCell<Page>, TaskIds)>;
 
 #[derive(Debug, Clone)]
 pub struct PageRc(PageRcInner);

From 72026aff06adeeaf729ed27d6b8a3e056d336b9a Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 3 Sep 2022 11:33:32 +0900
Subject: [PATCH 1735/3199] save

---
 scheduler/src/lib.rs | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index ebbecfc5168258..4eeb8d530c835c 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -822,11 +822,11 @@ impl ScheduleStage {
                     trace!("move to contended due to lock failure [{}/{}/{}]", unlockable_count, provisional_count, lock_count);
                     next_task.mark_as_contended();
                     *contended_count = contended_count.checked_add(1).unwrap();
-                    //for lock_attempt in next_task.tx.1.iter() {
-                    //    lock_attempt.contended_unique_weights().insert_task(unique_weight, Task::clone_in_queue(&a2));
-                    //}
-                    let a = Task::clone_in_queue(&next_task);
-                    task_sender.send((a, std::mem::take(&mut *next_task.for_indexer.0.borrow_mut()))).unwrap();
+                    for lock_attempt in next_task.tx.1.iter() {
+                        lock_attempt.contended_unique_weights().insert_task(unique_weight, Task::clone_in_queue(&a2));
+                    }
+                    //let a = Task::clone_in_queue(&next_task);
+                    //task_sender.send((a, std::mem::take(&mut *next_task.for_indexer.0.borrow_mut()))).unwrap();
 
                     // maybe run lightweight prune logic on contended_queue here.
                 } else {

From 1917a9aa0191b25b2804aa7224b9a04ec7b1c0d0 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 3 Sep 2022 11:33:52 +0900
Subject: [PATCH 1736/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 4eeb8d530c835c..5d30883620a8c9 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -822,7 +822,7 @@ impl ScheduleStage {
                     trace!("move to contended due to lock failure [{}/{}/{}]", unlockable_count, provisional_count, lock_count);
                     next_task.mark_as_contended();
                     *contended_count = contended_count.checked_add(1).unwrap();
-                    for lock_attempt in next_task.tx.1.iter() {
+                    for lock_attempt in next_task.lock_attempts().iter() {
                         lock_attempt.contended_unique_weights().insert_task(unique_weight, Task::clone_in_queue(&a2));
                     }
                     //let a = Task::clone_in_queue(&next_task);

From 87bc001327e7f611c591a339deb02f3225c2fa9f Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 3 Sep 2022 11:33:59 +0900
Subject: [PATCH 1737/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 5d30883620a8c9..04dbf5627cbee4 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -822,7 +822,7 @@ impl ScheduleStage {
                     trace!("move to contended due to lock failure [{}/{}/{}]", unlockable_count, provisional_count, lock_count);
                     next_task.mark_as_contended();
                     *contended_count = contended_count.checked_add(1).unwrap();
-                    for lock_attempt in next_task.lock_attempts().iter() {
+                    for lock_attempt in next_task.lock_attempts_mut().iter() {
                         lock_attempt.contended_unique_weights().insert_task(unique_weight, Task::clone_in_queue(&a2));
                     }
                     //let a = Task::clone_in_queue(&next_task);

From 98a2a46cc8d316558635032ecf621c2301ff7053 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 3 Sep 2022 11:34:06 +0900
Subject: [PATCH 1738/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 04dbf5627cbee4..b94f8604549d47 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -822,7 +822,7 @@ impl ScheduleStage {
                     trace!("move to contended due to lock failure [{}/{}/{}]", unlockable_count, provisional_count, lock_count);
                     next_task.mark_as_contended();
                     *contended_count = contended_count.checked_add(1).unwrap();
-                    for lock_attempt in next_task.lock_attempts_mut().iter() {
+                    for lock_attempt in next_task.lock_attempts_mut(ast).iter() {
                         lock_attempt.contended_unique_weights().insert_task(unique_weight, Task::clone_in_queue(&a2));
                     }
                     //let a = Task::clone_in_queue(&next_task);

From 9442651f9dd2b1d3a67ea2a693bf9bf89745fa26 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 3 Sep 2022 11:34:19 +0900
Subject: [PATCH 1739/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index b94f8604549d47..30f329c69a9a8e 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -823,7 +823,7 @@ impl ScheduleStage {
                     next_task.mark_as_contended();
                     *contended_count = contended_count.checked_add(1).unwrap();
                     for lock_attempt in next_task.lock_attempts_mut(ast).iter() {
-                        lock_attempt.contended_unique_weights().insert_task(unique_weight, Task::clone_in_queue(&a2));
+                        lock_attempt.contended_unique_weights().insert_task(unique_weight, Task::clone_in_queue(&next_task));
                     }
                     //let a = Task::clone_in_queue(&next_task);
                     //task_sender.send((a, std::mem::take(&mut *next_task.for_indexer.0.borrow_mut()))).unwrap();

From 2d21d19a1df01f6f0c2c2c5dc0e3af3e1ec9ee05 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 3 Sep 2022 11:34:48 +0900
Subject: [PATCH 1740/3199] save

---
 scheduler/src/lib.rs                    | 2 +-
 transaction-scheduler-bench/src/main.rs | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 30f329c69a9a8e..1fb21be1c5c6df 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -996,7 +996,7 @@ impl ScheduleStage {
     fn commit_completed_execution<AST: AtScheduleThread>(ast: AST, ee: &mut ExecutionEnvironment, address_book: &mut AddressBook, commit_time: &mut usize, provisioning_tracker_count: &mut usize) {
         // do par()-ly?
 
-        //ee.reindex();
+        ee.reindex();
         ee.task.record_commit_time(*commit_time);
         //ee.task.trace_timestamps("commit");
         //*commit_time = commit_time.checked_add(1).unwrap();
diff --git a/transaction-scheduler-bench/src/main.rs b/transaction-scheduler-bench/src/main.rs
index f8c3bc64f729ed..d32b196a027cbe 100644
--- a/transaction-scheduler-bench/src/main.rs
+++ b/transaction-scheduler-bench/src/main.rs
@@ -360,7 +360,7 @@ fn handle_transaction_batch(
         .priority_collected
         .fetch_add(priority_collected, Ordering::Relaxed);
 
-    transaction_batch.reindex();
+    //transaction_batch.reindex();
     completed_transaction_sender
         .send(transaction_batch)
         .unwrap();

From 33e6ed244644ac26e323dccb914a80806001cee1 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 3 Sep 2022 11:37:48 +0900
Subject: [PATCH 1741/3199] save

---
 scheduler/src/lib.rs                    | 13 +++++++------
 transaction-scheduler-bench/src/main.rs |  2 +-
 2 files changed, 8 insertions(+), 7 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 1fb21be1c5c6df..0e1a42087649a1 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -822,11 +822,12 @@ impl ScheduleStage {
                     trace!("move to contended due to lock failure [{}/{}/{}]", unlockable_count, provisional_count, lock_count);
                     next_task.mark_as_contended();
                     *contended_count = contended_count.checked_add(1).unwrap();
-                    for lock_attempt in next_task.lock_attempts_mut(ast).iter() {
-                        lock_attempt.contended_unique_weights().insert_task(unique_weight, Task::clone_in_queue(&next_task));
-                    }
-                    //let a = Task::clone_in_queue(&next_task);
-                    //task_sender.send((a, std::mem::take(&mut *next_task.for_indexer.0.borrow_mut()))).unwrap();
+
+                    //for lock_attempt in next_task.lock_attempts_mut(ast).iter() {
+                    //    lock_attempt.contended_unique_weights().insert_task(unique_weight, Task::clone_in_queue(&next_task));
+                    //}
+                    let a = Task::clone_in_queue(&next_task);
+                    task_sender.send((a, std::mem::take(&mut *next_task.for_indexer.0.borrow_mut()))).unwrap();
 
                     // maybe run lightweight prune logic on contended_queue here.
                 } else {
@@ -996,7 +997,7 @@ impl ScheduleStage {
     fn commit_completed_execution<AST: AtScheduleThread>(ast: AST, ee: &mut ExecutionEnvironment, address_book: &mut AddressBook, commit_time: &mut usize, provisioning_tracker_count: &mut usize) {
         // do par()-ly?
 
-        ee.reindex();
+        //ee.reindex();
         ee.task.record_commit_time(*commit_time);
         //ee.task.trace_timestamps("commit");
         //*commit_time = commit_time.checked_add(1).unwrap();
diff --git a/transaction-scheduler-bench/src/main.rs b/transaction-scheduler-bench/src/main.rs
index d32b196a027cbe..f8c3bc64f729ed 100644
--- a/transaction-scheduler-bench/src/main.rs
+++ b/transaction-scheduler-bench/src/main.rs
@@ -360,7 +360,7 @@ fn handle_transaction_batch(
         .priority_collected
         .fetch_add(priority_collected, Ordering::Relaxed);
 
-    //transaction_batch.reindex();
+    transaction_batch.reindex();
     completed_transaction_sender
         .send(transaction_batch)
         .unwrap();

From dbc54940b5648505da6e32c5347458e7d580236e Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 3 Sep 2022 11:43:56 +0900
Subject: [PATCH 1742/3199] save

---
 scheduler/src/lib.rs | 14 +++++++++-----
 1 file changed, 9 insertions(+), 5 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 0e1a42087649a1..91c1f93a6f5c0f 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -611,6 +611,14 @@ impl Task {
         assert!(!self.already_finished() && !self.currently_contended());
         self.uncontended.store(3, std::sync::atomic::Ordering::SeqCst)
     }
+
+    fn index(&self, task_sender: usize) {
+        //for lock_attempt in self.lock_attempts_mut(ast).iter() {
+        //    lock_attempt.contended_unique_weights().insert_task(unique_weight, Task::clone_in_queue(&self));
+        //}
+        let a = Task::clone_in_queue(&self);
+        task_sender.send((a, std::mem::take(&mut *self.for_indexer.0.borrow_mut()))).unwrap();
+    }
 }
 
 // RunnableQueue, ContendedQueue?
@@ -823,11 +831,7 @@ impl ScheduleStage {
                     next_task.mark_as_contended();
                     *contended_count = contended_count.checked_add(1).unwrap();
 
-                    //for lock_attempt in next_task.lock_attempts_mut(ast).iter() {
-                    //    lock_attempt.contended_unique_weights().insert_task(unique_weight, Task::clone_in_queue(&next_task));
-                    //}
-                    let a = Task::clone_in_queue(&next_task);
-                    task_sender.send((a, std::mem::take(&mut *next_task.for_indexer.0.borrow_mut()))).unwrap();
+                    next_task.index();
 
                     // maybe run lightweight prune logic on contended_queue here.
                 } else {

From 988a052a223798f4eab88db337671e28b12e0082 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 3 Sep 2022 11:44:14 +0900
Subject: [PATCH 1743/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 91c1f93a6f5c0f..178b1d7a1d173d 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -616,7 +616,7 @@ impl Task {
         //for lock_attempt in self.lock_attempts_mut(ast).iter() {
         //    lock_attempt.contended_unique_weights().insert_task(unique_weight, Task::clone_in_queue(&self));
         //}
-        let a = Task::clone_in_queue(&self);
+        let a = Task::clone_in_queue(self);
         task_sender.send((a, std::mem::take(&mut *self.for_indexer.0.borrow_mut()))).unwrap();
     }
 }

From 8970bf414c1b579a0d5e1510959f0deca6a11e60 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 3 Sep 2022 11:44:45 +0900
Subject: [PATCH 1744/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 178b1d7a1d173d..9423cd6a2cc3ae 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -612,7 +612,7 @@ impl Task {
         self.uncontended.store(3, std::sync::atomic::Ordering::SeqCst)
     }
 
-    fn index(&self, task_sender: usize) {
+    fn index(&TaskInQueue<self>, task_sender: usize) {
         //for lock_attempt in self.lock_attempts_mut(ast).iter() {
         //    lock_attempt.contended_unique_weights().insert_task(unique_weight, Task::clone_in_queue(&self));
         //}

From e02feeafbe861aad4ca743bb7cf5b9e8f69d807a Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 3 Sep 2022 11:45:17 +0900
Subject: [PATCH 1745/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 9423cd6a2cc3ae..55e45d943a7d52 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -612,7 +612,7 @@ impl Task {
         self.uncontended.store(3, std::sync::atomic::Ordering::SeqCst)
     }
 
-    fn index(&TaskInQueue<self>, task_sender: usize) {
+    fn index(self: &TaskInQueue<Self>, task_sender: usize) {
         //for lock_attempt in self.lock_attempts_mut(ast).iter() {
         //    lock_attempt.contended_unique_weights().insert_task(unique_weight, Task::clone_in_queue(&self));
         //}

From a0f0d2ebf9a7abb1179a4b868f6ffee488c56d1e Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 3 Sep 2022 11:45:30 +0900
Subject: [PATCH 1746/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 55e45d943a7d52..e9dc0215766202 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -612,7 +612,7 @@ impl Task {
         self.uncontended.store(3, std::sync::atomic::Ordering::SeqCst)
     }
 
-    fn index(self: &TaskInQueue<Self>, task_sender: usize) {
+    fn index(self: &TaskInQueue, task_sender: usize) {
         //for lock_attempt in self.lock_attempts_mut(ast).iter() {
         //    lock_attempt.contended_unique_weights().insert_task(unique_weight, Task::clone_in_queue(&self));
         //}

From a1f5deb514b6a3cdb7d92c02eadf3d50a9fe6b61 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 3 Sep 2022 11:46:16 +0900
Subject: [PATCH 1747/3199] save

---
 scheduler/src/lib.rs | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index e9dc0215766202..56cf86da911444 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -612,12 +612,12 @@ impl Task {
         self.uncontended.store(3, std::sync::atomic::Ordering::SeqCst)
     }
 
-    fn index(self: &TaskInQueue, task_sender: usize) {
+    fn index(this: &TaskInQueue, task_sender: usize) {
         //for lock_attempt in self.lock_attempts_mut(ast).iter() {
         //    lock_attempt.contended_unique_weights().insert_task(unique_weight, Task::clone_in_queue(&self));
         //}
-        let a = Task::clone_in_queue(self);
-        task_sender.send((a, std::mem::take(&mut *self.for_indexer.0.borrow_mut()))).unwrap();
+        let a = Task::clone_in_queue(this);
+        task_sender.send((a, std::mem::take(&mut *this.for_indexer.0.borrow_mut()))).unwrap();
     }
 }
 

From f90a2055f9d3468f21fe176a45ff48f002a4c2e2 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 3 Sep 2022 11:46:32 +0900
Subject: [PATCH 1748/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 56cf86da911444..1311171d11d585 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -831,7 +831,7 @@ impl ScheduleStage {
                     next_task.mark_as_contended();
                     *contended_count = contended_count.checked_add(1).unwrap();
 
-                    next_task.index();
+                    Task::index(next_task);
 
                     // maybe run lightweight prune logic on contended_queue here.
                 } else {

From e0024d8d8f94a1e963ec865e01fb65e170e27e3a Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 3 Sep 2022 11:46:48 +0900
Subject: [PATCH 1749/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 1311171d11d585..c24619882bf31c 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -831,7 +831,7 @@ impl ScheduleStage {
                     next_task.mark_as_contended();
                     *contended_count = contended_count.checked_add(1).unwrap();
 
-                    Task::index(next_task);
+                    Task::index(&next_task, task_sender);
 
                     // maybe run lightweight prune logic on contended_queue here.
                 } else {

From a00f7a0248a0cf38002153ce3a460694b0f478e3 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 3 Sep 2022 11:47:08 +0900
Subject: [PATCH 1750/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index c24619882bf31c..afa5f37e3a938d 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -612,7 +612,7 @@ impl Task {
         self.uncontended.store(3, std::sync::atomic::Ordering::SeqCst)
     }
 
-    fn index(this: &TaskInQueue, task_sender: usize) {
+    fn index(this: &TaskInQueue, task_sender: &crossbeam_channel::Sender<(triomphe::Arc<Task>, Vec<LockAttempt>)>) {
         //for lock_attempt in self.lock_attempts_mut(ast).iter() {
         //    lock_attempt.contended_unique_weights().insert_task(unique_weight, Task::clone_in_queue(&self));
         //}

From a7a9f0bd758c77e480289f97018c65edf6eaaf08 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 3 Sep 2022 11:47:54 +0900
Subject: [PATCH 1751/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index afa5f37e3a938d..e584e5182f3774 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -612,7 +612,7 @@ impl Task {
         self.uncontended.store(3, std::sync::atomic::Ordering::SeqCst)
     }
 
-    fn index(this: &TaskInQueue, task_sender: &crossbeam_channel::Sender<(triomphe::Arc<Task>, Vec<LockAttempt>)>) {
+    fn index(this: &TaskInQueue, task_sender: &crossbeam_channel::Sender<(TaskInQueue, Vec<LockAttempt>)>) {
         //for lock_attempt in self.lock_attempts_mut(ast).iter() {
         //    lock_attempt.contended_unique_weights().insert_task(unique_weight, Task::clone_in_queue(&self));
         //}

From d7dedfb3c7e39a4e910cecc7f2ef0064809a5206 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 3 Sep 2022 11:48:13 +0900
Subject: [PATCH 1752/3199] save

---
 scheduler/src/lib.rs | 1 +
 1 file changed, 1 insertion(+)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index e584e5182f3774..babdf530ce699f 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -612,6 +612,7 @@ impl Task {
         self.uncontended.store(3, std::sync::atomic::Ordering::SeqCst)
     }
 
+    #[inline(never)]
     fn index(this: &TaskInQueue, task_sender: &crossbeam_channel::Sender<(TaskInQueue, Vec<LockAttempt>)>) {
         //for lock_attempt in self.lock_attempts_mut(ast).iter() {
         //    lock_attempt.contended_unique_weights().insert_task(unique_weight, Task::clone_in_queue(&self));

From 0943092baf760698d6dc98e10acf89d40d31087a Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 3 Sep 2022 11:54:26 +0900
Subject: [PATCH 1753/3199] save

---
 scheduler/src/lib.rs | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index babdf530ce699f..807a1c1481e3c5 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -920,18 +920,18 @@ impl ScheduleStage {
 
     #[inline(never)]
     fn unlock_after_execution<AST: AtScheduleThread>(ast: AST, address_book: &mut AddressBook, lock_attempts: &mut Vec<LockAttempt>, provisioning_tracker_count: &mut usize) {
-        for mut l in lock_attempts.into_iter() {
+        for mut l in lock_attempts.iter_mut() {
             let newly_uncontended = address_book.reset_lock(ast, &mut l, true);
 
             let mut page = l.target.page_mut(ast);
             if newly_uncontended && page.next_usage == Usage::Unused {
-                let mut inserted = false;
+                //let mut inserted = false;
 
                 if let Some(task) = l.heaviest_uncontended.take() {
                     //assert!(!task.already_finished());
                     if /*true ||*/ task.currently_contended() {
                         //assert!(task.currently_contended());
-                        inserted = true;
+                        //inserted = true;
                         address_book.uncontended_task_ids.insert(task.unique_weight, task);
                     } /*else {
                         let contended_unique_weights = &page.contended_unique_weights;

From 2d4363b691ba99799e4dd7615de6bee34b46e83b Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 3 Sep 2022 11:54:46 +0900
Subject: [PATCH 1754/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 807a1c1481e3c5..565102a11ce445 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -919,7 +919,7 @@ impl ScheduleStage {
     }
 
     #[inline(never)]
-    fn unlock_after_execution<AST: AtScheduleThread>(ast: AST, address_book: &mut AddressBook, lock_attempts: &mut Vec<LockAttempt>, provisioning_tracker_count: &mut usize) {
+    fn unlock_after_execution<AST: AtScheduleThread>(ast: AST, address_book: &mut AddressBook, lock_attempts: &mut [LockAttempt], provisioning_tracker_count: &mut usize) {
         for mut l in lock_attempts.iter_mut() {
             let newly_uncontended = address_book.reset_lock(ast, &mut l, true);
 

From 6bcccd257acf7ae21871bb683659b9f86be4e487 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 3 Sep 2022 11:57:43 +0900
Subject: [PATCH 1755/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 565102a11ce445..01e30a070f9bbc 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -924,7 +924,7 @@ impl ScheduleStage {
             let newly_uncontended = address_book.reset_lock(ast, &mut l, true);
 
             let mut page = l.target.page_mut(ast);
-            if newly_uncontended && page.next_usage == Usage::Unused {
+            if newly_uncontended && page.next_usage == Usage::Unused && l.maybe_currently_contended() {
                 //let mut inserted = false;
 
                 if let Some(task) = l.heaviest_uncontended.take() {

From 0a8cfae45790deeeb2799eee984537b6dfef627a Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 3 Sep 2022 11:58:25 +0900
Subject: [PATCH 1756/3199] save

---
 scheduler/src/lib.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 01e30a070f9bbc..f4ddd9fb3fdd62 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -924,10 +924,10 @@ impl ScheduleStage {
             let newly_uncontended = address_book.reset_lock(ast, &mut l, true);
 
             let mut page = l.target.page_mut(ast);
-            if newly_uncontended && page.next_usage == Usage::Unused && l.maybe_currently_contended() {
+            if newly_uncontended && page.next_usage == Usage::Unused {
                 //let mut inserted = false;
 
-                if let Some(task) = l.heaviest_uncontended.take() {
+                if let Some(task) = l.heaviest_uncontended {
                     //assert!(!task.already_finished());
                     if /*true ||*/ task.currently_contended() {
                         //assert!(task.currently_contended());

From 6fc1a475b4497bb86feb39aa284aa49dffd6e373 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 3 Sep 2022 11:58:32 +0900
Subject: [PATCH 1757/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index f4ddd9fb3fdd62..95bc73f212a311 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -927,7 +927,7 @@ impl ScheduleStage {
             if newly_uncontended && page.next_usage == Usage::Unused {
                 //let mut inserted = false;
 
-                if let Some(task) = l.heaviest_uncontended {
+                if let Some(task) = &l.heaviest_uncontended {
                     //assert!(!task.already_finished());
                     if /*true ||*/ task.currently_contended() {
                         //assert!(task.currently_contended());

From bd5e31895ed2f518ac1507f3a3da7c289bb3836e Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 3 Sep 2022 11:59:01 +0900
Subject: [PATCH 1758/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 95bc73f212a311..b8e1acd2c5c882 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -932,7 +932,7 @@ impl ScheduleStage {
                     if /*true ||*/ task.currently_contended() {
                         //assert!(task.currently_contended());
                         //inserted = true;
-                        address_book.uncontended_task_ids.insert(task.unique_weight, task);
+                        address_book.uncontended_task_ids.insert(task.unique_weight, Task::clone_in_queue(task));
                     } /*else {
                         let contended_unique_weights = &page.contended_unique_weights;
                         contended_unique_weights.heaviest_task_cursor().map(|mut task_cursor| {

From 035aac838d1ef2c9e33f63d48db4d4f1c192ecf4 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 3 Sep 2022 12:00:48 +0900
Subject: [PATCH 1759/3199] save

---
 scheduler/src/lib.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index b8e1acd2c5c882..565102a11ce445 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -927,12 +927,12 @@ impl ScheduleStage {
             if newly_uncontended && page.next_usage == Usage::Unused {
                 //let mut inserted = false;
 
-                if let Some(task) = &l.heaviest_uncontended {
+                if let Some(task) = l.heaviest_uncontended.take() {
                     //assert!(!task.already_finished());
                     if /*true ||*/ task.currently_contended() {
                         //assert!(task.currently_contended());
                         //inserted = true;
-                        address_book.uncontended_task_ids.insert(task.unique_weight, Task::clone_in_queue(task));
+                        address_book.uncontended_task_ids.insert(task.unique_weight, task);
                     } /*else {
                         let contended_unique_weights = &page.contended_unique_weights;
                         contended_unique_weights.heaviest_task_cursor().map(|mut task_cursor| {

From 56e2e639099fba21fdc366cea4a84f74f0611c60 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 3 Sep 2022 12:01:54 +0900
Subject: [PATCH 1760/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 565102a11ce445..d69adf1f01cee0 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -920,7 +920,7 @@ impl ScheduleStage {
 
     #[inline(never)]
     fn unlock_after_execution<AST: AtScheduleThread>(ast: AST, address_book: &mut AddressBook, lock_attempts: &mut [LockAttempt], provisioning_tracker_count: &mut usize) {
-        for mut l in lock_attempts.iter_mut() {
+        for mut l in lock_attempts {
             let newly_uncontended = address_book.reset_lock(ast, &mut l, true);
 
             let mut page = l.target.page_mut(ast);

From 17d84a632b8d69f975e0007456ceaf4e78900cbe Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 3 Sep 2022 12:02:31 +0900
Subject: [PATCH 1761/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index d69adf1f01cee0..9d3f2dea495441 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -910,7 +910,7 @@ impl ScheduleStage {
         ast: AST,
         address_book: &mut AddressBook,
         unique_weight: &UniqueWeight,
-        lock_attempts: &mut Vec<LockAttempt>,
+        lock_attempts: &mut [LockAttempt],
         from_runnable: bool,
     ) {
         for l in lock_attempts {

From f60c3f61c2d2262984dac97d188e022ce818e0a6 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 3 Sep 2022 12:14:06 +0900
Subject: [PATCH 1762/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 9d3f2dea495441..0a49e42f632325 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1008,7 +1008,7 @@ impl ScheduleStage {
         //*commit_time = commit_time.checked_add(1).unwrap();
 
         // which order for data race free?: unlocking / marking
-        Self::unlock_after_execution(ast, address_book, &mut ee.finalized_lock_attempts, provisioning_tracker_count);
+        Self::unlock_after_execution(ast, address_book, &mut ee.finalized_lock_attempts, provisioning_tracker_count, ee.cu);
         ee.task.mark_as_finished();
 
         // block-wide qos validation will be done here

From 5af7adc90398a9a64044fc1ca69bfd2808429e6d Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 3 Sep 2022 12:17:25 +0900
Subject: [PATCH 1763/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 0a49e42f632325..d4371424464227 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -919,7 +919,7 @@ impl ScheduleStage {
     }
 
     #[inline(never)]
-    fn unlock_after_execution<AST: AtScheduleThread>(ast: AST, address_book: &mut AddressBook, lock_attempts: &mut [LockAttempt], provisioning_tracker_count: &mut usize) {
+    fn unlock_after_execution<AST: AtScheduleThread>(ast: AST, address_book: &mut AddressBook, lock_attempts: &mut [LockAttempt], provisioning_tracker_count: &mut usize, cu: usize) {
         for mut l in lock_attempts {
             let newly_uncontended = address_book.reset_lock(ast, &mut l, true);
 

From e37634838e51ea2667391783293be3f519598ab2 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 3 Sep 2022 12:18:59 +0900
Subject: [PATCH 1764/3199] save

---
 scheduler/src/lib.rs | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index d4371424464227..f8f7da5a88c413 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -923,6 +923,9 @@ impl ScheduleStage {
         for mut l in lock_attempts {
             let newly_uncontended = address_book.reset_lock(ast, &mut l, true);
 
+            l.cu += cu;
+
+
             let mut page = l.target.page_mut(ast);
             if newly_uncontended && page.next_usage == Usage::Unused {
                 //let mut inserted = false;

From f8e99b483c034a3f09258a8862c20f7503dbda20 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 3 Sep 2022 12:19:14 +0900
Subject: [PATCH 1765/3199] save

---
 scheduler/src/lib.rs | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index f8f7da5a88c413..c794f5befd3b04 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -923,10 +923,8 @@ impl ScheduleStage {
         for mut l in lock_attempts {
             let newly_uncontended = address_book.reset_lock(ast, &mut l, true);
 
-            l.cu += cu;
-
-
             let mut page = l.target.page_mut(ast);
+            page.cu += cu;
             if newly_uncontended && page.next_usage == Usage::Unused {
                 //let mut inserted = false;
 

From ed7bb8f64bddd0b1eba0539de162fede494342cc Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 3 Sep 2022 12:19:25 +0900
Subject: [PATCH 1766/3199] save

---
 scheduler/src/lib.rs | 1 +
 1 file changed, 1 insertion(+)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index c794f5befd3b04..4c4cc33f919fca 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -209,6 +209,7 @@ pub struct Page {
     current_usage: Usage,
     next_usage: Usage,
     provisional_task_ids: Vec<triomphe::Arc<ProvisioningTracker>>,
+    cu: usize,
     //loaded account from Accounts db
     //comulative_cu for qos; i.e. track serialized cumulative keyed by addresses and bail out block
     //producing as soon as any one of cu from the executing thread reaches to the limit

From e8e7c78c004368f91b1df08bafb1e3c430837efc Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 3 Sep 2022 12:19:39 +0900
Subject: [PATCH 1767/3199] save

---
 scheduler/src/lib.rs | 1 +
 1 file changed, 1 insertion(+)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 4c4cc33f919fca..12e3eb1ad9160e 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -221,6 +221,7 @@ impl Page {
             current_usage,
             next_usage: Usage::Unused,
             provisional_task_ids: Default::default(),
+            cu: Default::default(),
         }
     }
 

From f3a16f8f359f6823c4c700bba7dc58b3239c6652 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 3 Sep 2022 12:21:44 +0900
Subject: [PATCH 1768/3199] save

---
 scheduler/src/lib.rs | 1 +
 1 file changed, 1 insertion(+)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 12e3eb1ad9160e..09a57bcfb00598 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -341,6 +341,7 @@ impl AddressBook {
                         }
                     },
                 }
+        page.cu
     }
 
     fn reset_lock<AST: AtScheduleThread>(&mut self, ast: AST, attempt: &mut LockAttempt, after_execution: bool) -> bool {

From 5820e803d1f69a654396c7d2c7978bf97d20676a Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 3 Sep 2022 12:23:33 +0900
Subject: [PATCH 1769/3199] save

---
 scheduler/src/lib.rs | 11 +++++++----
 1 file changed, 7 insertions(+), 4 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 09a57bcfb00598..915b4cdd4d4dd3 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -282,7 +282,7 @@ impl AddressBook {
         prefer_immediate: bool,
         unique_weight: &UniqueWeight,
         attempt: &mut LockAttempt,
-    ) {
+    ) -> usize {
         let LockAttempt {target, requested_usage, status/*, remembered*/, ..} = attempt;
 
                 let mut page = target.page_mut(ast);
@@ -674,13 +674,16 @@ fn attempt_lock_for_execution<'a, AST: AtScheduleThread>(
     unique_weight: &UniqueWeight,
     message_hash: &'a Hash,
     placeholder_attempts: &mut Vec<LockAttempt>,
-) -> (usize, usize) {
+) -> (usize, usize, usize) {
     // no short-cuircuit; we at least all need to add to the contended queue
     let mut unlockable_count = 0;
     let mut provisional_count = 0;
+    let mut busiest_page_cu = 0;
 
     for attempt in placeholder_attempts.iter_mut() {
-        AddressBook::attempt_lock_address(ast, from_runnable, prefer_immediate, unique_weight, attempt);
+        let cu = AddressBook::attempt_lock_address(ast, from_runnable, prefer_immediate, unique_weight, attempt);
+        busiest_page_cu = busiest_page_cu.max(cu);
+
         match attempt.status {
             LockStatus::Succeded => {},
             LockStatus::Failed => {
@@ -692,7 +695,7 @@ fn attempt_lock_for_execution<'a, AST: AtScheduleThread>(
         }
     }
 
-    (unlockable_count, provisional_count)
+    (unlockable_count, provisional_count, busiest_page_cu)
 }
 
 type PreprocessedTransaction = (SanitizedTransaction, Vec<LockAttempt>);

From 91b83a338330e7ae997496e143e7d8be3581c494 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 3 Sep 2022 12:23:50 +0900
Subject: [PATCH 1770/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 915b4cdd4d4dd3..0c1fba3745421d 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -811,7 +811,7 @@ impl ScheduleStage {
             // plumb message_hash into StatusCache or implmenent our own for duplicate tx
             // detection?
 
-            let (unlockable_count, provisional_count) = attempt_lock_for_execution(
+            let (unlockable_count, provisional_count, busiest_page_cu) = attempt_lock_for_execution(
                 ast,
                 from_runnable,
                 prefer_immediate,

From 68c8d7a8d72e097cf55ef51116dddf83ca5d650d Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 3 Sep 2022 12:25:37 +0900
Subject: [PATCH 1771/3199] save

---
 scheduler/src/lib.rs | 18 ++++++++++--------
 1 file changed, 10 insertions(+), 8 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 0c1fba3745421d..7c29a7d606c720 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -674,15 +674,13 @@ fn attempt_lock_for_execution<'a, AST: AtScheduleThread>(
     unique_weight: &UniqueWeight,
     message_hash: &'a Hash,
     placeholder_attempts: &mut Vec<LockAttempt>,
-) -> (usize, usize, usize) {
+) -> (usize, usize) {
     // no short-cuircuit; we at least all need to add to the contended queue
     let mut unlockable_count = 0;
     let mut provisional_count = 0;
-    let mut busiest_page_cu = 0;
 
     for attempt in placeholder_attempts.iter_mut() {
-        let cu = AddressBook::attempt_lock_address(ast, from_runnable, prefer_immediate, unique_weight, attempt);
-        busiest_page_cu = busiest_page_cu.max(cu);
+        AddressBook::attempt_lock_address(ast, from_runnable, prefer_immediate, unique_weight, attempt);
 
         match attempt.status {
             LockStatus::Succeded => {},
@@ -695,7 +693,7 @@ fn attempt_lock_for_execution<'a, AST: AtScheduleThread>(
         }
     }
 
-    (unlockable_count, provisional_count, busiest_page_cu)
+    (unlockable_count, provisional_count)
 }
 
 type PreprocessedTransaction = (SanitizedTransaction, Vec<LockAttempt>);
@@ -811,7 +809,7 @@ impl ScheduleStage {
             // plumb message_hash into StatusCache or implmenent our own for duplicate tx
             // detection?
 
-            let (unlockable_count, provisional_count, busiest_page_cu) = attempt_lock_for_execution(
+            let (unlockable_count, provisional_count) = attempt_lock_for_execution(
                 ast,
                 from_runnable,
                 prefer_immediate,
@@ -918,10 +916,14 @@ impl ScheduleStage {
         unique_weight: &UniqueWeight,
         lock_attempts: &mut [LockAttempt],
         from_runnable: bool,
-    ) {
+    ) -> usize {
+        let mut busiest_page_cu = 0;
         for l in lock_attempts {
-            address_book.reset_lock(ast, l, false);
+            let cu = address_book.reset_lock(ast, l, false);
+            busiest_page_cu = busiest_page_cu.max(cu);
         }
+
+        busiest_page_cu
     }
 
     #[inline(never)]

From 75985817e6a0d815f4a031932b0827d785b55282 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 3 Sep 2022 12:26:57 +0900
Subject: [PATCH 1772/3199] save

---
 scheduler/src/lib.rs | 16 +++++++---------
 1 file changed, 7 insertions(+), 9 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 7c29a7d606c720..915b4cdd4d4dd3 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -674,13 +674,15 @@ fn attempt_lock_for_execution<'a, AST: AtScheduleThread>(
     unique_weight: &UniqueWeight,
     message_hash: &'a Hash,
     placeholder_attempts: &mut Vec<LockAttempt>,
-) -> (usize, usize) {
+) -> (usize, usize, usize) {
     // no short-cuircuit; we at least all need to add to the contended queue
     let mut unlockable_count = 0;
     let mut provisional_count = 0;
+    let mut busiest_page_cu = 0;
 
     for attempt in placeholder_attempts.iter_mut() {
-        AddressBook::attempt_lock_address(ast, from_runnable, prefer_immediate, unique_weight, attempt);
+        let cu = AddressBook::attempt_lock_address(ast, from_runnable, prefer_immediate, unique_weight, attempt);
+        busiest_page_cu = busiest_page_cu.max(cu);
 
         match attempt.status {
             LockStatus::Succeded => {},
@@ -693,7 +695,7 @@ fn attempt_lock_for_execution<'a, AST: AtScheduleThread>(
         }
     }
 
-    (unlockable_count, provisional_count)
+    (unlockable_count, provisional_count, busiest_page_cu)
 }
 
 type PreprocessedTransaction = (SanitizedTransaction, Vec<LockAttempt>);
@@ -916,14 +918,10 @@ impl ScheduleStage {
         unique_weight: &UniqueWeight,
         lock_attempts: &mut [LockAttempt],
         from_runnable: bool,
-    ) -> usize {
-        let mut busiest_page_cu = 0;
+    ) {
         for l in lock_attempts {
-            let cu = address_book.reset_lock(ast, l, false);
-            busiest_page_cu = busiest_page_cu.max(cu);
+            address_book.reset_lock(ast, l, false);
         }
-
-        busiest_page_cu
     }
 
     #[inline(never)]

From f3de6d04b7889eab27fe0e8070e3ebc2cfe54b66 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 3 Sep 2022 12:27:31 +0900
Subject: [PATCH 1773/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 915b4cdd4d4dd3..0c1fba3745421d 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -811,7 +811,7 @@ impl ScheduleStage {
             // plumb message_hash into StatusCache or implmenent our own for duplicate tx
             // detection?
 
-            let (unlockable_count, provisional_count) = attempt_lock_for_execution(
+            let (unlockable_count, provisional_count, busiest_page_cu) = attempt_lock_for_execution(
                 ast,
                 from_runnable,
                 prefer_immediate,

From 94e83fa2dc445446736014d557b5c5300858a5f0 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 3 Sep 2022 12:28:18 +0900
Subject: [PATCH 1774/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 0c1fba3745421d..1423b85046cede 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -811,7 +811,7 @@ impl ScheduleStage {
             // plumb message_hash into StatusCache or implmenent our own for duplicate tx
             // detection?
 
-            let (unlockable_count, provisional_count, busiest_page_cu) = attempt_lock_for_execution(
+            let (unlockable_count, provisional_count, _busiest_page_cu) = attempt_lock_for_execution(
                 ast,
                 from_runnable,
                 prefer_immediate,

From d8b1775abffdfa82af23e8de1445713acdfd76f2 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 3 Sep 2022 12:29:47 +0900
Subject: [PATCH 1775/3199] save

---
 scheduler/src/lib.rs | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 1423b85046cede..37c94143ac5fed 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -27,11 +27,12 @@ pub struct PageRc(PageRcInner);
 unsafe impl Send for PageRc {}
 unsafe impl Sync for PageRc {}
 
+type CU = u64;
 
 #[derive(Debug)]
 pub struct ExecutionEnvironment {
     //accounts: Vec<i8>,
-    pub cu: usize,
+    pub cu: CU,
     pub unique_weight: UniqueWeight,
     pub task: TaskInQueue,
     pub finalized_lock_attempts: Vec<LockAttempt>,

From d4a6f67626a2a321c5c1d3f93ba1bb4e4510cb0c Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 3 Sep 2022 12:30:00 +0900
Subject: [PATCH 1776/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 37c94143ac5fed..08d2bd960da1ad 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -926,7 +926,7 @@ impl ScheduleStage {
     }
 
     #[inline(never)]
-    fn unlock_after_execution<AST: AtScheduleThread>(ast: AST, address_book: &mut AddressBook, lock_attempts: &mut [LockAttempt], provisioning_tracker_count: &mut usize, cu: usize) {
+    fn unlock_after_execution<AST: AtScheduleThread>(ast: AST, address_book: &mut AddressBook, lock_attempts: &mut [LockAttempt], provisioning_tracker_count: &mut usize, cu: CU) {
         for mut l in lock_attempts {
             let newly_uncontended = address_book.reset_lock(ast, &mut l, true);
 

From cdeb9b249e5aefc754a9b8cda919924628a35d91 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 3 Sep 2022 12:30:19 +0900
Subject: [PATCH 1777/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 08d2bd960da1ad..9557b82c182c81 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -210,7 +210,7 @@ pub struct Page {
     current_usage: Usage,
     next_usage: Usage,
     provisional_task_ids: Vec<triomphe::Arc<ProvisioningTracker>>,
-    cu: usize,
+    cu: CU,
     //loaded account from Accounts db
     //comulative_cu for qos; i.e. track serialized cumulative keyed by addresses and bail out block
     //producing as soon as any one of cu from the executing thread reaches to the limit

From b062f3e2c10bcd9179c6ed0cbe72508345ef2c9c Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 3 Sep 2022 12:30:27 +0900
Subject: [PATCH 1778/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 9557b82c182c81..0001d8c6e89235 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -283,7 +283,7 @@ impl AddressBook {
         prefer_immediate: bool,
         unique_weight: &UniqueWeight,
         attempt: &mut LockAttempt,
-    ) -> usize {
+    ) -> CU {
         let LockAttempt {target, requested_usage, status/*, remembered*/, ..} = attempt;
 
                 let mut page = target.page_mut(ast);

From 760e806cccab7cb166cbd852197f7cb7e1490eda Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 3 Sep 2022 12:30:42 +0900
Subject: [PATCH 1779/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 0001d8c6e89235..a613fcc43fae74 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -675,7 +675,7 @@ fn attempt_lock_for_execution<'a, AST: AtScheduleThread>(
     unique_weight: &UniqueWeight,
     message_hash: &'a Hash,
     placeholder_attempts: &mut Vec<LockAttempt>,
-) -> (usize, usize, usize) {
+) -> (usize, usize, CU) {
     // no short-cuircuit; we at least all need to add to the contended queue
     let mut unlockable_count = 0;
     let mut provisional_count = 0;

From 0ee0e9c40f08f6053f4140839b5f8dfb21650cfb Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 3 Sep 2022 12:34:38 +0900
Subject: [PATCH 1780/3199] save

---
 scheduler/src/lib.rs | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index a613fcc43fae74..ac3015d6af1b6e 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -518,6 +518,10 @@ impl Task {
         self.tx.1.0.borrow()
     }
 
+    fn update_busiest_page_cu(&self, cu: CU) {
+        self.busiest_page_cu.store(cu, std::sync::atomic::Ordering::SeqCst);
+    }
+
     pub fn record_sequence_time(&self, clock: usize) {
         //self.sequence_time.store(clock, std::sync::atomic::Ordering::SeqCst);
     }
@@ -812,7 +816,7 @@ impl ScheduleStage {
             // plumb message_hash into StatusCache or implmenent our own for duplicate tx
             // detection?
 
-            let (unlockable_count, provisional_count, _busiest_page_cu) = attempt_lock_for_execution(
+            let (unlockable_count, provisional_count, busiest_page_cu) = attempt_lock_for_execution(
                 ast,
                 from_runnable,
                 prefer_immediate,
@@ -821,6 +825,7 @@ impl ScheduleStage {
                 &message_hash,
                 &mut next_task.lock_attempts_mut(ast),
             );
+            next_task.update_busiest_page_cu(busiest_page_cu);
 
             if unlockable_count > 0 {
                 //trace!("reset_lock_for_failed_execution(): {:?} {}", (&unique_weight, from_runnable), next_task.tx.0.signature());

From fc4e5c70f0e1c99b4aa403238da031731000dbfb Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 3 Sep 2022 12:35:24 +0900
Subject: [PATCH 1781/3199] save

---
 scheduler/src/lib.rs | 1 +
 1 file changed, 1 insertion(+)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index ac3015d6af1b6e..dd98995580bb5c 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -461,6 +461,7 @@ pub struct Task {
     unique_weight: UniqueWeight,
     pub tx: (SanitizedTransaction, LockAttemptsInCell), // actually should be Bundle
     pub contention_count: std::sync::atomic::AtomicUsize,
+    pub busiest_page_cu: std::sync::atomic::AtomicU64,
     pub uncontended: std::sync::atomic::AtomicUsize,
     pub sequence_time: std::sync::atomic::AtomicUsize,
     pub sequence_end_time: std::sync::atomic::AtomicUsize,

From 5b908a6cb19ea5c656e90c030a9c3ddaf52cb30c Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 3 Sep 2022 12:35:39 +0900
Subject: [PATCH 1782/3199] save

---
 scheduler/src/lib.rs | 1 +
 1 file changed, 1 insertion(+)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index dd98995580bb5c..612424b5548a0e 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -496,6 +496,7 @@ impl Task {
             unique_weight,
             tx: (tx.0, LockAttemptsInCell::new(std::cell::RefCell::new(tx.1))),
             contention_count: Default::default(),
+            busiest_page_cu: Default::default(),
             uncontended: Default::default(),
             sequence_time: std::sync::atomic::AtomicUsize::new(usize::max_value()),
             sequence_end_time: std::sync::atomic::AtomicUsize::new(usize::max_value()),

From dd4380d7a47740b2943c84b8b7a4624c3ef16ffb Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 3 Sep 2022 12:35:53 +0900
Subject: [PATCH 1783/3199] save

---
 scheduler/src/lib.rs | 1 +
 1 file changed, 1 insertion(+)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 612424b5548a0e..5fdc3b34da244a 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -590,6 +590,7 @@ impl Task {
             for_indexer: LockAttemptsInCell::new(std::cell::RefCell::new(self.lock_attempts_not_mut(nast).iter().map(|a| a.clone_for_test()).collect())),
             tx: (self.tx.0.clone(), LockAttemptsInCell::new(std::cell::RefCell::new(self.lock_attempts_not_mut(nast).iter().map(|l| l.clone_for_test()).collect::<Vec<_>>()))),
             contention_count: Default::default(),
+            busiest_page_cu: Default::default(),
             uncontended: Default::default(),
             sequence_time: std::sync::atomic::AtomicUsize::new(usize::max_value()),
             sequence_end_time: std::sync::atomic::AtomicUsize::new(usize::max_value()),

From 96f3d8a8f1d1326789fb68888d3ac7814963d641 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 3 Sep 2022 12:39:13 +0900
Subject: [PATCH 1784/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 5fdc3b34da244a..88604c35e96589 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -828,7 +828,6 @@ impl ScheduleStage {
                 &message_hash,
                 &mut next_task.lock_attempts_mut(ast),
             );
-            next_task.update_busiest_page_cu(busiest_page_cu);
 
             if unlockable_count > 0 {
                 //trace!("reset_lock_for_failed_execution(): {:?} {}", (&unique_weight, from_runnable), next_task.tx.0.signature());
@@ -858,6 +857,7 @@ impl ScheduleStage {
                 if from_runnable {
                     continue;
                 } else {
+                    next_task.update_busiest_page_cu(busiest_page_cu);
                     return None;
                 }
             } else if provisional_count > 0 {

From 07d408cc8a019587e4b4ff26d8f10210198c3d54 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 3 Sep 2022 12:39:51 +0900
Subject: [PATCH 1785/3199] save

---
 scheduler/src/lib.rs | 1 +
 1 file changed, 1 insertion(+)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 88604c35e96589..8270704ada85a8 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -857,6 +857,7 @@ impl ScheduleStage {
                 if from_runnable {
                     continue;
                 } else {
+                    // remove this task from stuck_tasks
                     next_task.update_busiest_page_cu(busiest_page_cu);
                     return None;
                 }

From 74760d1536d5a513cfd658be60ff7a017b8218d3 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 3 Sep 2022 12:40:01 +0900
Subject: [PATCH 1786/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 8270704ada85a8..3de170547a03e2 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -857,7 +857,7 @@ impl ScheduleStage {
                 if from_runnable {
                     continue;
                 } else {
-                    // remove this task from stuck_tasks
+                    // todo: remove this task from stuck_tasks
                     next_task.update_busiest_page_cu(busiest_page_cu);
                     return None;
                 }

From ffaeabe54b389ade4103ac42c5957e922d2bf98a Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 3 Sep 2022 12:41:00 +0900
Subject: [PATCH 1787/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 3de170547a03e2..487e35aba5b09d 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -857,7 +857,7 @@ impl ScheduleStage {
                 if from_runnable {
                     continue;
                 } else {
-                    // todo: remove this task from stuck_tasks
+                    // todo: remove this task from stuck_tasks before update_busiest_page_cu
                     next_task.update_busiest_page_cu(busiest_page_cu);
                     return None;
                 }

From 8883651874b57622a9d0b12aee8a88cbe01a3261 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 3 Sep 2022 12:48:30 +0900
Subject: [PATCH 1788/3199] save

---
 scheduler/src/lib.rs | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 487e35aba5b09d..88aa0af16b0da7 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1028,6 +1028,8 @@ impl ScheduleStage {
         Self::unlock_after_execution(ast, address_book, &mut ee.finalized_lock_attempts, provisioning_tracker_count, ee.cu);
         ee.task.mark_as_finished();
 
+        address_book.stuck_tasks.remove(ee.task_stuck_key());
+
         // block-wide qos validation will be done here
         // if error risen..:
         //   don't commit the tx for banking and potentially finish scheduling at block max cu

From 9d1c5672046cf9978d82e05f6f26b33b641723a8 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 3 Sep 2022 12:48:52 +0900
Subject: [PATCH 1789/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 88aa0af16b0da7..120fc30755b35e 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1028,7 +1028,7 @@ impl ScheduleStage {
         Self::unlock_after_execution(ast, address_book, &mut ee.finalized_lock_attempts, provisioning_tracker_count, ee.cu);
         ee.task.mark_as_finished();
 
-        address_book.stuck_tasks.remove(ee.task_stuck_key());
+        address_book.stuck_tasks.remove(ee.task.key_for_stuck_queue());
 
         // block-wide qos validation will be done here
         // if error risen..:

From a1261ac08ca2278c6717b062061213098e2b6260 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 3 Sep 2022 12:50:14 +0900
Subject: [PATCH 1790/3199] save

---
 scheduler/src/lib.rs | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 120fc30755b35e..e66ca1d2840bd4 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -631,6 +631,10 @@ impl Task {
         let a = Task::clone_in_queue(this);
         task_sender.send((a, std::mem::take(&mut *this.for_indexer.0.borrow_mut()))).unwrap();
     }
+
+    fn key_for_stuck_queue(&self) -> (CU, TaskId) {
+        (self.busiest_page_cu.load(std::sync::atomic::Ordering::SeqCst), self.unique_weight)
+    }
 }
 
 // RunnableQueue, ContendedQueue?

From fcc458c8a08d23564bad8ecb7cbec0ee7d9ea160 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 3 Sep 2022 12:52:05 +0900
Subject: [PATCH 1791/3199] save

---
 scheduler/src/lib.rs | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index e66ca1d2840bd4..eca3a457d54891 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -239,12 +239,15 @@ type WeightedTaskIds = std::collections::BTreeMap<TaskId, TaskInQueue>;
 //type AddressMapEntry<'a, K, V> = std::collections::hash_map::Entry<'a, K, V>;
 type AddressMapEntry<'a> = dashmap::mapref::entry::Entry<'a, Pubkey, PageRc>;
 
+type StuckTaskId = (CU, TaskId);
+
 // needs ttl mechanism and prune
 #[derive(Default)]
 pub struct AddressBook {
     book: AddressMap,
     uncontended_task_ids: WeightedTaskIds,
     fulfilled_provisional_task_ids: WeightedTaskIds,
+    stuck_tasks: std::collections::BTreeMap<StuckTaskId, TaskInQueue>,
 }
 
 #[derive(Debug)]
@@ -632,7 +635,7 @@ impl Task {
         task_sender.send((a, std::mem::take(&mut *this.for_indexer.0.borrow_mut()))).unwrap();
     }
 
-    fn key_for_stuck_queue(&self) -> (CU, TaskId) {
+    fn task_id_as_stuck(&self) -> StuckTaskId {
         (self.busiest_page_cu.load(std::sync::atomic::Ordering::SeqCst), self.unique_weight)
     }
 }

From 6404191dbb36afc11de084f8f5d81925b606b8c9 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 3 Sep 2022 12:52:14 +0900
Subject: [PATCH 1792/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index eca3a457d54891..b35db2ebca95c9 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1035,7 +1035,7 @@ impl ScheduleStage {
         Self::unlock_after_execution(ast, address_book, &mut ee.finalized_lock_attempts, provisioning_tracker_count, ee.cu);
         ee.task.mark_as_finished();
 
-        address_book.stuck_tasks.remove(ee.task.key_for_stuck_queue());
+        address_book.stuck_tasks.remove(ee.task.task_id_as_stuck());
 
         // block-wide qos validation will be done here
         // if error risen..:

From f33ac5bb4c72208245a612872f88f162ca81b434 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 3 Sep 2022 12:52:35 +0900
Subject: [PATCH 1793/3199] save

---
 scheduler/src/lib.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index b35db2ebca95c9..e19735540c5901 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -635,7 +635,7 @@ impl Task {
         task_sender.send((a, std::mem::take(&mut *this.for_indexer.0.borrow_mut()))).unwrap();
     }
 
-    fn task_id_as_stuck(&self) -> StuckTaskId {
+    fn stuck_task_id(&self) -> StuckTaskId {
         (self.busiest_page_cu.load(std::sync::atomic::Ordering::SeqCst), self.unique_weight)
     }
 }
@@ -1035,7 +1035,7 @@ impl ScheduleStage {
         Self::unlock_after_execution(ast, address_book, &mut ee.finalized_lock_attempts, provisioning_tracker_count, ee.cu);
         ee.task.mark_as_finished();
 
-        address_book.stuck_tasks.remove(ee.task.task_id_as_stuck());
+        address_book.stuck_tasks.remove(ee.task.stuck_task_id());
 
         // block-wide qos validation will be done here
         // if error risen..:

From aff9f64440f8169d78f6429e34cd918debeeaa60 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 3 Sep 2022 12:52:47 +0900
Subject: [PATCH 1794/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index e19735540c5901..886b4303ca87c3 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1035,7 +1035,7 @@ impl ScheduleStage {
         Self::unlock_after_execution(ast, address_book, &mut ee.finalized_lock_attempts, provisioning_tracker_count, ee.cu);
         ee.task.mark_as_finished();
 
-        address_book.stuck_tasks.remove(ee.task.stuck_task_id());
+        address_book.stuck_tasks.remove(&ee.task.stuck_task_id());
 
         // block-wide qos validation will be done here
         // if error risen..:

From 9bc509e3bf3c0262137d9b0a0d71d852135f744f Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 3 Sep 2022 12:53:50 +0900
Subject: [PATCH 1795/3199] save

---
 scheduler/src/lib.rs | 1 +
 1 file changed, 1 insertion(+)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 886b4303ca87c3..0505b7e658399b 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -865,6 +865,7 @@ impl ScheduleStage {
                     continue;
                 } else {
                     // todo: remove this task from stuck_tasks before update_busiest_page_cu
+                    address_book.stuck_tasks.remove(&next_task.stuck_task_id());
                     next_task.update_busiest_page_cu(busiest_page_cu);
                     return None;
                 }

From 4ff4afbdde04fbb494eb208457ab2e8b666894e6 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 3 Sep 2022 12:54:20 +0900
Subject: [PATCH 1796/3199] save

---
 scheduler/src/lib.rs | 1 +
 1 file changed, 1 insertion(+)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 0505b7e658399b..166670fe0dc2ca 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -867,6 +867,7 @@ impl ScheduleStage {
                     // todo: remove this task from stuck_tasks before update_busiest_page_cu
                     address_book.stuck_tasks.remove(&next_task.stuck_task_id());
                     next_task.update_busiest_page_cu(busiest_page_cu);
+                    address_book.stuck_tasks.insert(next_task.stuck_task_id(), Task::clone_in_queue(next_task));
                     return None;
                 }
             } else if provisional_count > 0 {

From d87275b1449b8a583ae69d7105dbba283bf5f151 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 3 Sep 2022 12:54:26 +0900
Subject: [PATCH 1797/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 166670fe0dc2ca..55343b5d197ee9 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -867,7 +867,7 @@ impl ScheduleStage {
                     // todo: remove this task from stuck_tasks before update_busiest_page_cu
                     address_book.stuck_tasks.remove(&next_task.stuck_task_id());
                     next_task.update_busiest_page_cu(busiest_page_cu);
-                    address_book.stuck_tasks.insert(next_task.stuck_task_id(), Task::clone_in_queue(next_task));
+                    address_book.stuck_tasks.insert(next_task.stuck_task_id(), Task::clone_in_queue(&next_task));
                     return None;
                 }
             } else if provisional_count > 0 {

From 9a1f13d0e7597f68c332f2bf073ca5c730668615 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 3 Sep 2022 12:54:56 +0900
Subject: [PATCH 1798/3199] save

---
 scheduler/src/lib.rs | 1 +
 1 file changed, 1 insertion(+)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 55343b5d197ee9..7de9223c2697b8 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -862,6 +862,7 @@ impl ScheduleStage {
                 }
 
                 if from_runnable {
+                    address_book.stuck_tasks.insert(next_task.stuck_task_id(), Task::clone_in_queue(&next_task));
                     continue;
                 } else {
                     // todo: remove this task from stuck_tasks before update_busiest_page_cu

From 8cf1d79afe835d6fa55ef3673f1b7754c20c6388 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 3 Sep 2022 12:57:27 +0900
Subject: [PATCH 1799/3199] save

---
 scheduler/src/lib.rs | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 7de9223c2697b8..2d2f49c27c6240 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1138,6 +1138,7 @@ impl ScheduleStage {
                 assert_eq!(task_receiver.len(), 0);
             }).unwrap();
         }
+        let mut start = Instant::new();
 
         let (mut from_disconnected, mut from_exec_disconnected) = (false, false);
         loop {
@@ -1179,6 +1180,10 @@ impl ScheduleStage {
             loop {
                 while (executing_queue_count + provisioning_tracker_count) < max_executing_queue_count {
                     trace!("schedule_once (from: {}, to: {}, runnnable: {}, contended: {}, (immediate+provisional)/max: ({}+{})/{}) active from contended: {}!", from.len(), to_execute_substage.len(), runnable_queue.task_count(), contended_count, executing_queue_count, provisioning_tracker_count, max_executing_queue_count, address_book.uncontended_task_ids.len());
+                    if start.elapsed > Duration::from_millis(100) {
+                        start = Instant::now();
+                        info!("schedule_once (from: {}, to: {}, runnnable: {}, contended: {}, (immediate+provisional)/max: ({}+{})/{}) active from contended: {}!", from.len(), to_execute_substage.len(), runnable_queue.task_count(), contended_count, executing_queue_count, provisioning_tracker_count, max_executing_queue_count, address_book.uncontended_task_ids.len());
+                    }
                     let prefer_immediate = provisioning_tracker_count/4 > executing_queue_count;
                     if let Some(ee) = Self::schedule_next_execution(ast, &task_sender, runnable_queue, address_book, &mut contended_count, prefer_immediate, &sequence_time, &mut queue_clock, &mut execute_clock, &mut provisioning_tracker_count) {
                         executing_queue_count = executing_queue_count.checked_add(1).unwrap();

From 688ead5c1af2e1265f697d9e127fde00a145f55d Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 3 Sep 2022 12:57:40 +0900
Subject: [PATCH 1800/3199] save

---
 scheduler/src/lib.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 2d2f49c27c6240..ecb2eb79eb52a7 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1138,7 +1138,7 @@ impl ScheduleStage {
                 assert_eq!(task_receiver.len(), 0);
             }).unwrap();
         }
-        let mut start = Instant::new();
+        let mut start = std::time::Instant::new();
 
         let (mut from_disconnected, mut from_exec_disconnected) = (false, false);
         loop {
@@ -1181,7 +1181,7 @@ impl ScheduleStage {
                 while (executing_queue_count + provisioning_tracker_count) < max_executing_queue_count {
                     trace!("schedule_once (from: {}, to: {}, runnnable: {}, contended: {}, (immediate+provisional)/max: ({}+{})/{}) active from contended: {}!", from.len(), to_execute_substage.len(), runnable_queue.task_count(), contended_count, executing_queue_count, provisioning_tracker_count, max_executing_queue_count, address_book.uncontended_task_ids.len());
                     if start.elapsed > Duration::from_millis(100) {
-                        start = Instant::now();
+                        start = std::time::Instant::now();
                         info!("schedule_once (from: {}, to: {}, runnnable: {}, contended: {}, (immediate+provisional)/max: ({}+{})/{}) active from contended: {}!", from.len(), to_execute_substage.len(), runnable_queue.task_count(), contended_count, executing_queue_count, provisioning_tracker_count, max_executing_queue_count, address_book.uncontended_task_ids.len());
                     }
                     let prefer_immediate = provisioning_tracker_count/4 > executing_queue_count;

From 2762cd5fe9150a58fe3d5fb5b298dffc37257b28 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 3 Sep 2022 12:57:50 +0900
Subject: [PATCH 1801/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index ecb2eb79eb52a7..43d3ea95edc4a1 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1138,7 +1138,7 @@ impl ScheduleStage {
                 assert_eq!(task_receiver.len(), 0);
             }).unwrap();
         }
-        let mut start = std::time::Instant::new();
+        let mut start = std::time::Instant::now();
 
         let (mut from_disconnected, mut from_exec_disconnected) = (false, false);
         loop {

From d68ebc0ecf15efe0aa59f8e17807b7f702437f03 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 3 Sep 2022 12:58:00 +0900
Subject: [PATCH 1802/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 43d3ea95edc4a1..4f50630c1627e4 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1180,7 +1180,7 @@ impl ScheduleStage {
             loop {
                 while (executing_queue_count + provisioning_tracker_count) < max_executing_queue_count {
                     trace!("schedule_once (from: {}, to: {}, runnnable: {}, contended: {}, (immediate+provisional)/max: ({}+{})/{}) active from contended: {}!", from.len(), to_execute_substage.len(), runnable_queue.task_count(), contended_count, executing_queue_count, provisioning_tracker_count, max_executing_queue_count, address_book.uncontended_task_ids.len());
-                    if start.elapsed > Duration::from_millis(100) {
+                    if start.elapsed() > Duration::from_millis(100) {
                         start = std::time::Instant::now();
                         info!("schedule_once (from: {}, to: {}, runnnable: {}, contended: {}, (immediate+provisional)/max: ({}+{})/{}) active from contended: {}!", from.len(), to_execute_substage.len(), runnable_queue.task_count(), contended_count, executing_queue_count, provisioning_tracker_count, max_executing_queue_count, address_book.uncontended_task_ids.len());
                     }

From 49b437a7735efdca055d71a503afa46fd7fa24b6 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 3 Sep 2022 12:58:08 +0900
Subject: [PATCH 1803/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 4f50630c1627e4..4c2308340d6c60 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1180,7 +1180,7 @@ impl ScheduleStage {
             loop {
                 while (executing_queue_count + provisioning_tracker_count) < max_executing_queue_count {
                     trace!("schedule_once (from: {}, to: {}, runnnable: {}, contended: {}, (immediate+provisional)/max: ({}+{})/{}) active from contended: {}!", from.len(), to_execute_substage.len(), runnable_queue.task_count(), contended_count, executing_queue_count, provisioning_tracker_count, max_executing_queue_count, address_book.uncontended_task_ids.len());
-                    if start.elapsed() > Duration::from_millis(100) {
+                    if start.elapsed() > std::time::Duration::from_millis(100) {
                         start = std::time::Instant::now();
                         info!("schedule_once (from: {}, to: {}, runnnable: {}, contended: {}, (immediate+provisional)/max: ({}+{})/{}) active from contended: {}!", from.len(), to_execute_substage.len(), runnable_queue.task_count(), contended_count, executing_queue_count, provisioning_tracker_count, max_executing_queue_count, address_book.uncontended_task_ids.len());
                     }

From 2f03b6ef9cd8336615056d195cbef83a3f0f9cd2 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 3 Sep 2022 13:00:36 +0900
Subject: [PATCH 1804/3199] save

---
 scheduler/src/lib.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 4c2308340d6c60..6bc00da6d0f6fb 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1179,10 +1179,10 @@ impl ScheduleStage {
 
             loop {
                 while (executing_queue_count + provisioning_tracker_count) < max_executing_queue_count {
-                    trace!("schedule_once (from: {}, to: {}, runnnable: {}, contended: {}, (immediate+provisional)/max: ({}+{})/{}) active from contended: {}!", from.len(), to_execute_substage.len(), runnable_queue.task_count(), contended_count, executing_queue_count, provisioning_tracker_count, max_executing_queue_count, address_book.uncontended_task_ids.len());
+                    trace!("schedule_once (from: {}, to: {}, runnnable: {}, contended: {}, (immediate+provisional)/max: ({}+{})/{}) active from contended: {} stuck: {}!", from.len(), to_execute_substage.len(), runnable_queue.task_count(), contended_count, executing_queue_count, provisioning_tracker_count, max_executing_queue_count, address_book.uncontended_task_ids.len(), address_book.stuck_tasks.len());
                     if start.elapsed() > std::time::Duration::from_millis(100) {
                         start = std::time::Instant::now();
-                        info!("schedule_once (from: {}, to: {}, runnnable: {}, contended: {}, (immediate+provisional)/max: ({}+{})/{}) active from contended: {}!", from.len(), to_execute_substage.len(), runnable_queue.task_count(), contended_count, executing_queue_count, provisioning_tracker_count, max_executing_queue_count, address_book.uncontended_task_ids.len());
+                        info!("schedule_once (from: {}, to: {}, runnnable: {}, contended: {}, (immediate+provisional)/max: ({}+{})/{}) active from contended: {} stuck: {}!", from.len(), to_execute_substage.len(), runnable_queue.task_count(), contended_count, executing_queue_count, provisioning_tracker_count, max_executing_queue_count, address_book.uncontended_task_ids.len(), address_book.stuck_tasks.len());
                     }
                     let prefer_immediate = provisioning_tracker_count/4 > executing_queue_count;
                     if let Some(ee) = Self::schedule_next_execution(ast, &task_sender, runnable_queue, address_book, &mut contended_count, prefer_immediate, &sequence_time, &mut queue_clock, &mut execute_clock, &mut provisioning_tracker_count) {

From 87ae4a7178e60ae75e82a5471736d458b943c0f2 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 3 Sep 2022 13:01:10 +0900
Subject: [PATCH 1805/3199] save

---
 scheduler/src/lib.rs | 1 +
 1 file changed, 1 insertion(+)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 6bc00da6d0f6fb..0eb184c60d3e0f 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1226,6 +1226,7 @@ impl ScheduleStage {
                 }
             }
         }
+        info!("schedule_once (from: {}, to: {}, runnnable: {}, contended: {}, (immediate+provisional)/max: ({}+{})/{}) active from contended: {} stuck: {}!", from.len(), to_execute_substage.len(), runnable_queue.task_count(), contended_count, executing_queue_count, provisioning_tracker_count, max_executing_queue_count, address_book.uncontended_task_ids.len(), address_book.stuck_tasks.len());
         info!("run finished...");
     }
 

From 44f2816e1a9e610b3269ff23920b5607ec0ca535 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 3 Sep 2022 13:01:17 +0900
Subject: [PATCH 1806/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 0eb184c60d3e0f..fe72ba7e5f1177 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1226,8 +1226,8 @@ impl ScheduleStage {
                 }
             }
         }
-        info!("schedule_once (from: {}, to: {}, runnnable: {}, contended: {}, (immediate+provisional)/max: ({}+{})/{}) active from contended: {} stuck: {}!", from.len(), to_execute_substage.len(), runnable_queue.task_count(), contended_count, executing_queue_count, provisioning_tracker_count, max_executing_queue_count, address_book.uncontended_task_ids.len(), address_book.stuck_tasks.len());
         info!("run finished...");
+        info!("schedule_once (from: {}, to: {}, runnnable: {}, contended: {}, (immediate+provisional)/max: ({}+{})/{}) active from contended: {} stuck: {}!", from.len(), to_execute_substage.len(), runnable_queue.task_count(), contended_count, executing_queue_count, provisioning_tracker_count, max_executing_queue_count, address_book.uncontended_task_ids.len(), address_book.stuck_tasks.len());
     }
 
     pub fn run(

From 8bd43c008bc3e725a4897bc59f97a54536534681 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 3 Sep 2022 13:04:23 +0900
Subject: [PATCH 1807/3199] save

---
 scheduler/src/lib.rs | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index fe72ba7e5f1177..73a650bc0f6ce7 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -866,7 +866,8 @@ impl ScheduleStage {
                     continue;
                 } else {
                     // todo: remove this task from stuck_tasks before update_busiest_page_cu
-                    address_book.stuck_tasks.remove(&next_task.stuck_task_id());
+                    let removed = address_book.stuck_tasks.remove(&next_task.stuck_task_id());
+                    assert!(removed);
                     next_task.update_busiest_page_cu(busiest_page_cu);
                     address_book.stuck_tasks.insert(next_task.stuck_task_id(), Task::clone_in_queue(&next_task));
                     return None;

From 2168621a959135df8bf7cdc06b519d318aa13198 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 3 Sep 2022 13:04:49 +0900
Subject: [PATCH 1808/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 73a650bc0f6ce7..9d287e44e3ef82 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -867,7 +867,7 @@ impl ScheduleStage {
                 } else {
                     // todo: remove this task from stuck_tasks before update_busiest_page_cu
                     let removed = address_book.stuck_tasks.remove(&next_task.stuck_task_id());
-                    assert!(removed);
+                    assert!(removed.is_some());
                     next_task.update_busiest_page_cu(busiest_page_cu);
                     address_book.stuck_tasks.insert(next_task.stuck_task_id(), Task::clone_in_queue(&next_task));
                     return None;

From a8b5ac947c713fed7ac812548bded6ce66dc0b05 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 3 Sep 2022 13:05:25 +0900
Subject: [PATCH 1809/3199] save

---
 scheduler/src/lib.rs | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 9d287e44e3ef82..6a8726c8d153e1 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -869,7 +869,8 @@ impl ScheduleStage {
                     let removed = address_book.stuck_tasks.remove(&next_task.stuck_task_id());
                     assert!(removed.is_some());
                     next_task.update_busiest_page_cu(busiest_page_cu);
-                    address_book.stuck_tasks.insert(next_task.stuck_task_id(), Task::clone_in_queue(&next_task));
+                    let a = address_book.stuck_tasks.insert(next_task.stuck_task_id(), Task::clone_in_queue(&next_task));
+                    assert!(a);
                     return None;
                 }
             } else if provisional_count > 0 {

From 554631da7e23f0932c323d85c511c4738667d3cf Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 3 Sep 2022 13:05:43 +0900
Subject: [PATCH 1810/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 6a8726c8d153e1..34aae5417ffdc5 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -870,7 +870,7 @@ impl ScheduleStage {
                     assert!(removed.is_some());
                     next_task.update_busiest_page_cu(busiest_page_cu);
                     let a = address_book.stuck_tasks.insert(next_task.stuck_task_id(), Task::clone_in_queue(&next_task));
-                    assert!(a);
+                    assert!(a.is_none());
                     return None;
                 }
             } else if provisional_count > 0 {

From ce10255fb49f162bd68fece7258f0eb94adcbe08 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 3 Sep 2022 13:08:32 +0900
Subject: [PATCH 1811/3199] save

---
 scheduler/src/lib.rs | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 34aae5417ffdc5..4eefd56817bc6f 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -760,12 +760,12 @@ impl ScheduleStage {
             (Some(heaviest_runnable_entry), None) => {
                 trace!("select: runnable only");
                 let t = heaviest_runnable_entry.remove();
-                return Some((true, t))
+                Some((true, t))
             }
             (None, Some(weight_from_contended)) => {
                 trace!("select: contended only");
                 let t = weight_from_contended.remove();
-                return Some(( false, t))
+                Some(( false, t))
             },
             (Some(heaviest_runnable_entry), Some(weight_from_contended)) => {
                 let weight_from_runnable = heaviest_runnable_entry.key();
@@ -774,11 +774,11 @@ impl ScheduleStage {
                 if weight_from_runnable > uw {
                     trace!("select: runnable > contended");
                     let t = heaviest_runnable_entry.remove();
-                    return Some((true, t))
+                    Some((true, t))
                 } else if uw > weight_from_runnable {
                     trace!("select: contended > runnnable");
                     let t = weight_from_contended.remove();
-                    return Some(( false, t))
+                    Some(( false, t))
                 } else {
                     unreachable!(
                         "identical unique weights shouldn't exist in both runnable and contended"
@@ -787,7 +787,7 @@ impl ScheduleStage {
             }
             (None, None) => {
                 trace!("select: none");
-                return None
+                None
             }
         }
     }

From c2414c05fc7fa5e9c868d137846c1142b0b76d4d Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 3 Sep 2022 13:11:10 +0900
Subject: [PATCH 1812/3199] save

---
 scheduler/src/lib.rs | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 4eefd56817bc6f..286b285b5b81f3 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -749,6 +749,7 @@ impl ScheduleStage {
     fn select_next_task<'a>(
         runnable_queue: &'a mut TaskQueue,
         address_book: &mut AddressBook,
+        contended_count: &usize,
     ) -> Option<(
         bool,
         TaskInQueue,
@@ -814,7 +815,7 @@ impl ScheduleStage {
 
         trace!("pop begin");
         loop {
-        if let Some((from_runnable, mut next_task)) = Self::select_next_task(runnable_queue, address_book) {
+        if let Some((from_runnable, mut next_task)) = Self::select_next_task(runnable_queue, address_book, contended_count) {
             trace!("pop loop iteration");
             if from_runnable {
                 next_task.record_queue_time(*sequence_clock, *queue_clock);

From 844d605a2b864afbcb99acc1086074056d0532b9 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 3 Sep 2022 13:11:56 +0900
Subject: [PATCH 1813/3199] save

---
 scheduler/src/lib.rs | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 286b285b5b81f3..941fd4cef73c50 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -788,7 +788,11 @@ impl ScheduleStage {
             }
             (None, None) => {
                 trace!("select: none");
-                None
+                if runnable_queue.len() == 0 && contended_count == 0 {
+                    todo!("handle stuck!");
+                } else {
+                    None
+                }
             }
         }
     }

From 70bd35685283e7f80f42a598dee4fd7c32b5b9ff Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 3 Sep 2022 13:12:03 +0900
Subject: [PATCH 1814/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 941fd4cef73c50..72f1a8f029f88f 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -788,7 +788,7 @@ impl ScheduleStage {
             }
             (None, None) => {
                 trace!("select: none");
-                if runnable_queue.len() == 0 && contended_count == 0 {
+                if runnable_queue.len() == 0 && *contended_count == 0 {
                     todo!("handle stuck!");
                 } else {
                     None

From be175b8b2ddadd4495a2f3b32b64237dfc9c95ac Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 3 Sep 2022 13:12:11 +0900
Subject: [PATCH 1815/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 72f1a8f029f88f..7e3f913159f5a4 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -788,7 +788,7 @@ impl ScheduleStage {
             }
             (None, None) => {
                 trace!("select: none");
-                if runnable_queue.len() == 0 && *contended_count == 0 {
+                if runnable_queue.count() == 0 && *contended_count == 0 {
                     todo!("handle stuck!");
                 } else {
                     None

From 91baf271b6cbd8534277bcc3ce2e4fb83bffa75f Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 3 Sep 2022 13:12:34 +0900
Subject: [PATCH 1816/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 7e3f913159f5a4..7763f5a0e07d6f 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -788,7 +788,7 @@ impl ScheduleStage {
             }
             (None, None) => {
                 trace!("select: none");
-                if runnable_queue.count() == 0 && *contended_count == 0 {
+                if runnable_queue.task_count() == 0 && *contended_count == 0 {
                     todo!("handle stuck!");
                 } else {
                     None

From 181f79d2740b7a790ba16f6fb25822f67fe6d622 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 3 Sep 2022 13:13:23 +0900
Subject: [PATCH 1817/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 7763f5a0e07d6f..4c81ff145988e6 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -788,7 +788,7 @@ impl ScheduleStage {
             }
             (None, None) => {
                 trace!("select: none");
-                if runnable_queue.task_count() == 0 && *contended_count == 0 {
+                if runnable_queue.task_count() == 0 && *contended_count == 0 && stuck_tasks.len() > 0 {
                     todo!("handle stuck!");
                 } else {
                     None

From c44f069f58527481f6c2ddf2b70ec900bdb0c7c8 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 3 Sep 2022 13:13:46 +0900
Subject: [PATCH 1818/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 4c81ff145988e6..65e37a84b98e2f 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -788,7 +788,7 @@ impl ScheduleStage {
             }
             (None, None) => {
                 trace!("select: none");
-                if runnable_queue.task_count() == 0 && *contended_count == 0 && stuck_tasks.len() > 0 {
+                if runnable_queue.task_count() == 0 && *contended_count == 0 && address_book.stuck_tasks.len() > 0 {
                     todo!("handle stuck!");
                 } else {
                     None

From bb4da812f53f4442d625f9c279c2bbb4715a4661 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 3 Sep 2022 13:14:42 +0900
Subject: [PATCH 1819/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 65e37a84b98e2f..39087d65c536b5 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -789,7 +789,7 @@ impl ScheduleStage {
             (None, None) => {
                 trace!("select: none");
                 if runnable_queue.task_count() == 0 && *contended_count == 0 && address_book.stuck_tasks.len() > 0 {
-                    todo!("handle stuck!");
+                    Some((false, address_book.stuck_tasks.pop_first().unwrap()))
                 } else {
                     None
                 }

From 3cf3049485d7a6ba3c1f07d9e6c318c30dd4f9b8 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 3 Sep 2022 13:15:01 +0900
Subject: [PATCH 1820/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 39087d65c536b5..8e708f94baf764 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -789,7 +789,7 @@ impl ScheduleStage {
             (None, None) => {
                 trace!("select: none");
                 if runnable_queue.task_count() == 0 && *contended_count == 0 && address_book.stuck_tasks.len() > 0 {
-                    Some((false, address_book.stuck_tasks.pop_first().unwrap()))
+                    Some((false, address_book.stuck_tasks.pop_first().unwrap().1))
                 } else {
                     None
                 }

From a5580cf28f8edc7e6e12dd442f6567ddd75e76db Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 3 Sep 2022 13:15:50 +0900
Subject: [PATCH 1821/3199] save

---
 scheduler/src/lib.rs | 1 +
 1 file changed, 1 insertion(+)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 8e708f94baf764..f504314f08ff87 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -789,6 +789,7 @@ impl ScheduleStage {
             (None, None) => {
                 trace!("select: none");
                 if runnable_queue.task_count() == 0 && *contended_count == 0 && address_book.stuck_tasks.len() > 0 {
+                    error!("handling stuck...");
                     Some((false, address_book.stuck_tasks.pop_first().unwrap().1))
                 } else {
                     None

From a05dfa0e0796a0712d71177740322a7cf6bc8d66 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 3 Sep 2022 13:18:01 +0900
Subject: [PATCH 1822/3199] save

---
 scheduler/src/lib.rs | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index f504314f08ff87..7683cacb004fbe 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -868,7 +868,8 @@ impl ScheduleStage {
                 }
 
                 if from_runnable {
-                    address_book.stuck_tasks.insert(next_task.stuck_task_id(), Task::clone_in_queue(&next_task));
+                    let a = address_book.stuck_tasks.insert(next_task.stuck_task_id(), Task::clone_in_queue(&next_task));
+                    assert!(a.is_none());
                     continue;
                 } else {
                     // todo: remove this task from stuck_tasks before update_busiest_page_cu

From 3cf56eec9a3d5c3de4e3375e9b19cc2ffef9e877 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 3 Sep 2022 13:20:04 +0900
Subject: [PATCH 1823/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 7683cacb004fbe..cbc77f57db5617 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -788,7 +788,7 @@ impl ScheduleStage {
             }
             (None, None) => {
                 trace!("select: none");
-                if runnable_queue.task_count() == 0 && *contended_count == 0 && address_book.stuck_tasks.len() > 0 {
+                if runnable_queue.task_count() == 0 && /**contended_count > 0 &&*/ address_book.stuck_tasks.len() > 0 {
                     error!("handling stuck...");
                     Some((false, address_book.stuck_tasks.pop_first().unwrap().1))
                 } else {

From 6df697ee8abc95dd18c93ded3b2aaeb8dc6a8373 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 3 Sep 2022 13:20:16 +0900
Subject: [PATCH 1824/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index cbc77f57db5617..54fa65439d5738 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -788,7 +788,7 @@ impl ScheduleStage {
             }
             (None, None) => {
                 trace!("select: none");
-                if runnable_queue.task_count() == 0 && /**contended_count > 0 &&*/ address_book.stuck_tasks.len() > 0 {
+                if runnable_queue.task_count() == 0 && /* *contended_count > 0 &&*/ address_book.stuck_tasks.len() > 0 {
                     error!("handling stuck...");
                     Some((false, address_book.stuck_tasks.pop_first().unwrap().1))
                 } else {

From fc2cd3eb52489ca54cc35c651a454978fded8e30 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 3 Sep 2022 13:22:20 +0900
Subject: [PATCH 1825/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 54fa65439d5738..68ad90701f8e25 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -903,7 +903,7 @@ impl ScheduleStage {
             trace!("successful lock: (from_runnable: {}) after {} contentions", from_runnable, next_task.contention_count.load(std::sync::atomic::Ordering::SeqCst));
 
             if !from_runnable {
-                *contended_count = contended_count.checked_sub(1).unwrap();
+                *contended_count = contended_count.checked_sub(1).unwrap_or_default();
                 next_task.mark_as_uncontended();
             }
             let lock_attempts = std::mem::take(&mut *next_task.lock_attempts_mut(ast));

From 4368fa79fa84945272db8fce9511b7e7825a495c Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 3 Sep 2022 13:23:37 +0900
Subject: [PATCH 1826/3199] save

---
 scheduler/src/lib.rs | 1 +
 1 file changed, 1 insertion(+)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 68ad90701f8e25..54b7e93d4a0a7e 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -902,6 +902,7 @@ impl ScheduleStage {
 
             trace!("successful lock: (from_runnable: {}) after {} contentions", from_runnable, next_task.contention_count.load(std::sync::atomic::Ordering::SeqCst));
 
+            assert!(!self.already_finished());
             if !from_runnable {
                 *contended_count = contended_count.checked_sub(1).unwrap_or_default();
                 next_task.mark_as_uncontended();

From e8e68dc2ed1fb2d4e72b7cbf606d77acde99e95c Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 3 Sep 2022 13:23:45 +0900
Subject: [PATCH 1827/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 54b7e93d4a0a7e..7c0a8d97ef8ddc 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -902,7 +902,7 @@ impl ScheduleStage {
 
             trace!("successful lock: (from_runnable: {}) after {} contentions", from_runnable, next_task.contention_count.load(std::sync::atomic::Ordering::SeqCst));
 
-            assert!(!self.already_finished());
+            assert!(!next_task.already_finished());
             if !from_runnable {
                 *contended_count = contended_count.checked_sub(1).unwrap_or_default();
                 next_task.mark_as_uncontended();

From fb7a1ffb40ff7cb19da246491fdcb4685ada989d Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 3 Sep 2022 13:25:10 +0900
Subject: [PATCH 1828/3199] save

---
 scheduler/src/lib.rs | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 7c0a8d97ef8ddc..1fa0fe7059c074 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -887,6 +887,9 @@ impl ScheduleStage {
                 trace!("provisional exec: [{}/{}]", provisional_count, lock_count);
                 *contended_count = contended_count.checked_sub(1).unwrap();
                 next_task.mark_as_uncontended();
+                let removed = address_book.stuck_tasks.remove(&next_task.stuck_task_id());
+                assert!(removed.is_some());
+
                 let tracker = triomphe::Arc::new(ProvisioningTracker::new(provisional_count, Task::clone_in_queue(&next_task)));
                 *provisioning_tracker_count = provisioning_tracker_count.checked_add(1).unwrap();
                 Self::finalize_lock_for_provisional_execution(

From 30e4f8456e00287d820136ac7c8bed0203105499 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 3 Sep 2022 13:25:55 +0900
Subject: [PATCH 1829/3199] save

---
 scheduler/src/lib.rs | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 1fa0fe7059c074..6643632f3aff11 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -887,8 +887,7 @@ impl ScheduleStage {
                 trace!("provisional exec: [{}/{}]", provisional_count, lock_count);
                 *contended_count = contended_count.checked_sub(1).unwrap();
                 next_task.mark_as_uncontended();
-                let removed = address_book.stuck_tasks.remove(&next_task.stuck_task_id());
-                assert!(removed.is_some());
+                address_book.stuck_tasks.remove(&next_task.stuck_task_id());
 
                 let tracker = triomphe::Arc::new(ProvisioningTracker::new(provisional_count, Task::clone_in_queue(&next_task)));
                 *provisioning_tracker_count = provisioning_tracker_count.checked_add(1).unwrap();

From 12af3524df7defa231371b40538c0e8464608448 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 3 Sep 2022 13:26:50 +0900
Subject: [PATCH 1830/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 6643632f3aff11..2e8dc43d9ce78b 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -906,7 +906,7 @@ impl ScheduleStage {
 
             assert!(!next_task.already_finished());
             if !from_runnable {
-                *contended_count = contended_count.checked_sub(1).unwrap_or_default();
+                *contended_count = contended_count.checked_sub(1).unwrap();
                 next_task.mark_as_uncontended();
             }
             let lock_attempts = std::mem::take(&mut *next_task.lock_attempts_mut(ast));

From a72993e9512a464a466120739992821534b7690b Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 3 Sep 2022 13:28:51 +0900
Subject: [PATCH 1831/3199] save

---
 scheduler/src/lib.rs | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 2e8dc43d9ce78b..2d83d3a5e2b3b9 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -873,10 +873,9 @@ impl ScheduleStage {
                     continue;
                 } else {
                     // todo: remove this task from stuck_tasks before update_busiest_page_cu
-                    let removed = address_book.stuck_tasks.remove(&next_task.stuck_task_id());
-                    assert!(removed.is_some());
+                    let removed = address_book.stuck_tasks.remove(&next_task.stuck_task_id()).unwrap();
                     next_task.update_busiest_page_cu(busiest_page_cu);
-                    let a = address_book.stuck_tasks.insert(next_task.stuck_task_id(), Task::clone_in_queue(&next_task));
+                    let a = address_book.stuck_tasks.insert(next_task.stuck_task_id(), removed);
                     assert!(a.is_none());
                     return None;
                 }

From 9fc6409a54306b6e4193e15b3456875b32496bf9 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 3 Sep 2022 13:29:55 +0900
Subject: [PATCH 1832/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 2d83d3a5e2b3b9..caad41971c615e 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -905,7 +905,7 @@ impl ScheduleStage {
 
             assert!(!next_task.already_finished());
             if !from_runnable {
-                *contended_count = contended_count.checked_sub(1).unwrap();
+                *contended_count = contended_count.checked_sub(1).unwrap_or_default();
                 next_task.mark_as_uncontended();
             }
             let lock_attempts = std::mem::take(&mut *next_task.lock_attempts_mut(ast));

From 4c45f32d59fa3d2cf78b1d14832c1dbb03fe856e Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 3 Sep 2022 13:31:13 +0900
Subject: [PATCH 1833/3199] save

---
 scheduler/src/lib.rs | 1 +
 1 file changed, 1 insertion(+)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index caad41971c615e..9724468a8c7ef8 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -868,6 +868,7 @@ impl ScheduleStage {
                 }
 
                 if from_runnable {
+                    next_task.update_busiest_page_cu(busiest_page_cu);
                     let a = address_book.stuck_tasks.insert(next_task.stuck_task_id(), Task::clone_in_queue(&next_task));
                     assert!(a.is_none());
                     continue;

From 682ef1a72590e7e4e1f41badb654ad42111a36ca Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 3 Sep 2022 13:32:27 +0900
Subject: [PATCH 1834/3199] save

---
 scheduler/src/lib.rs | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 9724468a8c7ef8..7ec4390832df27 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -636,7 +636,9 @@ impl Task {
     }
 
     fn stuck_task_id(&self) -> StuckTaskId {
-        (self.busiest_page_cu.load(std::sync::atomic::Ordering::SeqCst), self.unique_weight)
+        let cu = self.busiest_page_cu.load(std::sync::atomic::Ordering::SeqCst);
+        assert_ne!(cu, 0);
+        (cu, self.unique_weight)
     }
 }
 
@@ -868,7 +870,7 @@ impl ScheduleStage {
                 }
 
                 if from_runnable {
-                    next_task.update_busiest_page_cu(busiest_page_cu);
+                    //next_task.update_busiest_page_cu(busiest_page_cu);
                     let a = address_book.stuck_tasks.insert(next_task.stuck_task_id(), Task::clone_in_queue(&next_task));
                     assert!(a.is_none());
                     continue;

From 076e7243c45318d9c1bfc0187008d32f163ea116 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 3 Sep 2022 13:33:01 +0900
Subject: [PATCH 1835/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 7ec4390832df27..2e9a6b4114dbfe 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -870,7 +870,7 @@ impl ScheduleStage {
                 }
 
                 if from_runnable {
-                    //next_task.update_busiest_page_cu(busiest_page_cu);
+                    next_task.update_busiest_page_cu(busiest_page_cu);
                     let a = address_book.stuck_tasks.insert(next_task.stuck_task_id(), Task::clone_in_queue(&next_task));
                     assert!(a.is_none());
                     continue;

From cbbdd187dd20239d6fe662cad9739183a52527b0 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 3 Sep 2022 13:33:40 +0900
Subject: [PATCH 1836/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 2e9a6b4114dbfe..d2497a9e1fa5e4 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -695,7 +695,7 @@ fn attempt_lock_for_execution<'a, AST: AtScheduleThread>(
     // no short-cuircuit; we at least all need to add to the contended queue
     let mut unlockable_count = 0;
     let mut provisional_count = 0;
-    let mut busiest_page_cu = 0;
+    let mut busiest_page_cu = 1;
 
     for attempt in placeholder_attempts.iter_mut() {
         let cu = AddressBook::attempt_lock_address(ast, from_runnable, prefer_immediate, unique_weight, attempt);

From 1d83f70025b857ec46db277642ebd97ec3d618ae Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 3 Sep 2022 13:35:05 +0900
Subject: [PATCH 1837/3199] save

---
 scheduler/src/lib.rs | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index d2497a9e1fa5e4..08f8de36d6e145 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -889,6 +889,7 @@ impl ScheduleStage {
                 trace!("provisional exec: [{}/{}]", provisional_count, lock_count);
                 *contended_count = contended_count.checked_sub(1).unwrap();
                 next_task.mark_as_uncontended();
+                next_task.update_busiest_page_cu(busiest_page_cu);
                 address_book.stuck_tasks.remove(&next_task.stuck_task_id());
 
                 let tracker = triomphe::Arc::new(ProvisioningTracker::new(provisional_count, Task::clone_in_queue(&next_task)));
@@ -912,6 +913,7 @@ impl ScheduleStage {
                 next_task.mark_as_uncontended();
             }
             let lock_attempts = std::mem::take(&mut *next_task.lock_attempts_mut(ast));
+            next_task.update_busiest_page_cu(busiest_page_cu);
 
             return Some((unique_weight, next_task, lock_attempts));
         } else {

From 5b23d147a0f74596a5801cd7703c9a5f5b374147 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 3 Sep 2022 13:36:02 +0900
Subject: [PATCH 1838/3199] save

---
 scheduler/src/lib.rs | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 08f8de36d6e145..f0d5c252c5a4b4 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -889,8 +889,8 @@ impl ScheduleStage {
                 trace!("provisional exec: [{}/{}]", provisional_count, lock_count);
                 *contended_count = contended_count.checked_sub(1).unwrap();
                 next_task.mark_as_uncontended();
-                next_task.update_busiest_page_cu(busiest_page_cu);
                 address_book.stuck_tasks.remove(&next_task.stuck_task_id());
+                next_task.update_busiest_page_cu(busiest_page_cu);
 
                 let tracker = triomphe::Arc::new(ProvisioningTracker::new(provisional_count, Task::clone_in_queue(&next_task)));
                 *provisioning_tracker_count = provisioning_tracker_count.checked_add(1).unwrap();
@@ -911,9 +911,10 @@ impl ScheduleStage {
             if !from_runnable {
                 *contended_count = contended_count.checked_sub(1).unwrap_or_default();
                 next_task.mark_as_uncontended();
+            } else {
+                next_task.update_busiest_page_cu(busiest_page_cu);
             }
             let lock_attempts = std::mem::take(&mut *next_task.lock_attempts_mut(ast));
-            next_task.update_busiest_page_cu(busiest_page_cu);
 
             return Some((unique_weight, next_task, lock_attempts));
         } else {

From 69debc4231899fef50141b0a7285ee91f6cd239b Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 3 Sep 2022 13:38:08 +0900
Subject: [PATCH 1839/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index f0d5c252c5a4b4..3151fb2446c463 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -909,7 +909,7 @@ impl ScheduleStage {
 
             assert!(!next_task.already_finished());
             if !from_runnable {
-                *contended_count = contended_count.checked_sub(1).unwrap_or_default();
+                *contended_count = contended_count.checked_sub(1).unwrap();
                 next_task.mark_as_uncontended();
             } else {
                 next_task.update_busiest_page_cu(busiest_page_cu);

From 4ff84567b9fe623747747fbd7e61229b220db2ac Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 3 Sep 2022 13:43:22 +0900
Subject: [PATCH 1840/3199] save

---
 scheduler/src/lib.rs | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 3151fb2446c463..1476e6a7dbd571 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -792,7 +792,9 @@ impl ScheduleStage {
                 trace!("select: none");
                 if runnable_queue.task_count() == 0 && /* *contended_count > 0 &&*/ address_book.stuck_tasks.len() > 0 {
                     error!("handling stuck...");
-                    Some((false, address_book.stuck_tasks.pop_first().unwrap().1))
+                    let task = address_book.stuck_tasks.pop_first().unwrap().1;
+                    task.mark_as_contended();
+                    Some((false, task))
                 } else {
                     None
                 }

From 8bf168c098944ef9651b4e04f5b725a0f5da3614 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 3 Sep 2022 13:44:43 +0900
Subject: [PATCH 1841/3199] save

---
 scheduler/src/lib.rs | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 1476e6a7dbd571..463cf2f67687cc 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -792,7 +792,8 @@ impl ScheduleStage {
                 trace!("select: none");
                 if runnable_queue.task_count() == 0 && /* *contended_count > 0 &&*/ address_book.stuck_tasks.len() > 0 {
                     error!("handling stuck...");
-                    let task = address_book.stuck_tasks.pop_first().unwrap().1;
+                    let (stuck_task_id, task) = address_book.stuck_tasks.pop_first().unwrap();
+                    assert_eq!(task.stuck_task_id(), stuck_task_id);
                     task.mark_as_contended();
                     Some((false, task))
                 } else {

From 3a93d8ca99a52390bbf1a35f189854ab09f0844b Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 3 Sep 2022 13:45:57 +0900
Subject: [PATCH 1842/3199] save

---
 scheduler/src/lib.rs | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 463cf2f67687cc..b555a8fa717d94 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -794,7 +794,9 @@ impl ScheduleStage {
                     error!("handling stuck...");
                     let (stuck_task_id, task) = address_book.stuck_tasks.pop_first().unwrap();
                     assert_eq!(task.stuck_task_id(), stuck_task_id);
-                    task.mark_as_contended();
+
+                    task.mark_as_contended(); // why this is needed???
+
                     Some((false, task))
                 } else {
                     None

From a2efe5f8c90643326f7c38681e7c9679ed8ad5d1 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 3 Sep 2022 13:47:30 +0900
Subject: [PATCH 1843/3199] save

---
 scheduler/src/lib.rs | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index b555a8fa717d94..2d3fcf8db29af5 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -795,9 +795,11 @@ impl ScheduleStage {
                     let (stuck_task_id, task) = address_book.stuck_tasks.pop_first().unwrap();
                     assert_eq!(task.stuck_task_id(), stuck_task_id);
 
-                    task.mark_as_contended(); // why this is needed???
-
-                    Some((false, task))
+                    if task.currently_contended() {
+                        Some((false, task))
+                    } else {
+                        None
+                    }
                 } else {
                     None
                 }

From caa99c5240d6ac4d7e37c7cf99d80275f6ce7e4e Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 3 Sep 2022 13:49:30 +0900
Subject: [PATCH 1844/3199] save

---
 scheduler/src/lib.rs | 1 +
 1 file changed, 1 insertion(+)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 2d3fcf8db29af5..88d00f2eccbb7f 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -793,6 +793,7 @@ impl ScheduleStage {
                 if runnable_queue.task_count() == 0 && /* *contended_count > 0 &&*/ address_book.stuck_tasks.len() > 0 {
                     error!("handling stuck...");
                     let (stuck_task_id, task) = address_book.stuck_tasks.pop_first().unwrap();
+                    // ensure proper rekeying
                     assert_eq!(task.stuck_task_id(), stuck_task_id);
 
                     if task.currently_contended() {

From 0eb34c0890f2ffed38df663c25f88cd6e0c58fd2 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 3 Sep 2022 13:50:46 +0900
Subject: [PATCH 1845/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 88d00f2eccbb7f..c91a5382722832 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -791,7 +791,7 @@ impl ScheduleStage {
             (None, None) => {
                 trace!("select: none");
                 if runnable_queue.task_count() == 0 && /* *contended_count > 0 &&*/ address_book.stuck_tasks.len() > 0 {
-                    error!("handling stuck...");
+                    trace!("handling stuck...");
                     let (stuck_task_id, task) = address_book.stuck_tasks.pop_first().unwrap();
                     // ensure proper rekeying
                     assert_eq!(task.stuck_task_id(), stuck_task_id);

From 6a51bdf5c691e49fee3f464f484bf05fa9a8baee Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 3 Sep 2022 13:53:02 +0900
Subject: [PATCH 1846/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index c91a5382722832..10e0b5b8ac55c3 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1205,7 +1205,7 @@ impl ScheduleStage {
             loop {
                 while (executing_queue_count + provisioning_tracker_count) < max_executing_queue_count {
                     trace!("schedule_once (from: {}, to: {}, runnnable: {}, contended: {}, (immediate+provisional)/max: ({}+{})/{}) active from contended: {} stuck: {}!", from.len(), to_execute_substage.len(), runnable_queue.task_count(), contended_count, executing_queue_count, provisioning_tracker_count, max_executing_queue_count, address_book.uncontended_task_ids.len(), address_book.stuck_tasks.len());
-                    if start.elapsed() > std::time::Duration::from_millis(100) {
+                    if start.elapsed() > std::time::Duration::from_millis(1000) {
                         start = std::time::Instant::now();
                         info!("schedule_once (from: {}, to: {}, runnnable: {}, contended: {}, (immediate+provisional)/max: ({}+{})/{}) active from contended: {} stuck: {}!", from.len(), to_execute_substage.len(), runnable_queue.task_count(), contended_count, executing_queue_count, provisioning_tracker_count, max_executing_queue_count, address_book.uncontended_task_ids.len(), address_book.stuck_tasks.len());
                     }

From 7dc7a4dbdf0b6d5fd4503704cde6819dd08d4593 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 3 Sep 2022 13:59:17 +0900
Subject: [PATCH 1847/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 10e0b5b8ac55c3..b2a5b4031aed21 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -638,7 +638,7 @@ impl Task {
     fn stuck_task_id(&self) -> StuckTaskId {
         let cu = self.busiest_page_cu.load(std::sync::atomic::Ordering::SeqCst);
         assert_ne!(cu, 0);
-        (cu, self.unique_weight)
+        (cu, TaskId::max_value() - self.unique_weight)
     }
 }
 

From 653b81e0e61535424bd15c55d8025f18166f87b1 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 3 Sep 2022 14:09:49 +0900
Subject: [PATCH 1848/3199] save

---
 scheduler/src/lib.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index b2a5b4031aed21..11b772f117d4ea 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -51,7 +51,7 @@ impl ExecutionEnvironment {
     //}
     //
     #[inline(never)]
-    pub fn reindex(&mut self) {
+    pub fn reindex_to_address_book(&mut self) {
         let uq = self.unique_weight;
         //self.task.trace_timestamps("in_exec(self)");
         let should_remove = self.task.contention_count.load(std::sync::atomic::Ordering::SeqCst) > 0;
@@ -627,7 +627,7 @@ impl Task {
     }
 
     #[inline(never)]
-    fn index(this: &TaskInQueue, task_sender: &crossbeam_channel::Sender<(TaskInQueue, Vec<LockAttempt>)>) {
+    fn index_to_address_book(this: &TaskInQueue, task_sender: &crossbeam_channel::Sender<(TaskInQueue, Vec<LockAttempt>)>) {
         //for lock_attempt in self.lock_attempts_mut(ast).iter() {
         //    lock_attempt.contended_unique_weights().insert_task(unique_weight, Task::clone_in_queue(&self));
         //}

From 915f57b607488b4791728778db9a696110dd477e Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 3 Sep 2022 14:09:58 +0900
Subject: [PATCH 1849/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 11b772f117d4ea..42dfb2ed2a4a0d 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -869,7 +869,7 @@ impl ScheduleStage {
                     next_task.mark_as_contended();
                     *contended_count = contended_count.checked_add(1).unwrap();
 
-                    Task::index(&next_task, task_sender);
+                    Task::index_to_address_book(&next_task, task_sender);
 
                     // maybe run lightweight prune logic on contended_queue here.
                 } else {

From 4edfa2f86fc1c078453c5d0d719c7184c48199e0 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 3 Sep 2022 14:10:12 +0900
Subject: [PATCH 1850/3199] save

---
 transaction-scheduler-bench/src/main.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/transaction-scheduler-bench/src/main.rs b/transaction-scheduler-bench/src/main.rs
index f8c3bc64f729ed..a31ad715507b30 100644
--- a/transaction-scheduler-bench/src/main.rs
+++ b/transaction-scheduler-bench/src/main.rs
@@ -360,7 +360,7 @@ fn handle_transaction_batch(
         .priority_collected
         .fetch_add(priority_collected, Ordering::Relaxed);
 
-    transaction_batch.reindex();
+    transaction_batch.reindex_to_address_book();
     completed_transaction_sender
         .send(transaction_batch)
         .unwrap();

From 16d9253b4222a516a272f3d7c012ffd425a29dd9 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 3 Sep 2022 14:34:26 +0900
Subject: [PATCH 1851/3199] save

---
 scheduler/src/lib.rs | 1 +
 1 file changed, 1 insertion(+)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 42dfb2ed2a4a0d..a2bcd329b9e32d 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -799,6 +799,7 @@ impl ScheduleStage {
                     if task.currently_contended() {
                         Some((false, task))
                     } else {
+                        // is it expected for uncontended taks is in the stuck queue?
                         None
                     }
                 } else {

From 04fbfe09286b1bead5991287e64f2ca10c28663b Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 3 Sep 2022 14:36:07 +0900
Subject: [PATCH 1852/3199] save

---
 scheduler/src/lib.rs | 817 ++++++++++++++++++++++++++-----------------
 1 file changed, 505 insertions(+), 312 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index a2bcd329b9e32d..1a2a14dea75801 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -54,42 +54,50 @@ impl ExecutionEnvironment {
     pub fn reindex_to_address_book(&mut self) {
         let uq = self.unique_weight;
         //self.task.trace_timestamps("in_exec(self)");
-        let should_remove = self.task.contention_count.load(std::sync::atomic::Ordering::SeqCst) > 0;
+        let should_remove = self
+            .task
+            .contention_count
+            .load(std::sync::atomic::Ordering::SeqCst)
+            > 0;
         for mut lock_attempt in self.finalized_lock_attempts.iter_mut() {
             let contended_unique_weights = lock_attempt.contended_unique_weights();
-            contended_unique_weights.heaviest_task_cursor().map(|mut task_cursor| {
-                let mut found = true;
-                let mut removed = false;
-                let mut task = task_cursor.value();
-                //task.trace_timestamps("in_exec(initial list)");
-                while !task.currently_contended() {
-                    if task_cursor.key() == &uq {
-                        assert!(should_remove);
-                        removed = task_cursor.remove();
-                        assert!(removed);
+            contended_unique_weights
+                .heaviest_task_cursor()
+                .map(|mut task_cursor| {
+                    let mut found = true;
+                    let mut removed = false;
+                    let mut task = task_cursor.value();
+                    //task.trace_timestamps("in_exec(initial list)");
+                    while !task.currently_contended() {
+                        if task_cursor.key() == &uq {
+                            assert!(should_remove);
+                            removed = task_cursor.remove();
+                            assert!(removed);
+                        }
+                        if task.already_finished() {
+                            task_cursor.remove();
+                        }
+                        if let Some(new_cursor) = task_cursor.prev() {
+                            assert!(new_cursor.key() < task_cursor.key());
+                            task_cursor = new_cursor;
+                            task = task_cursor.value();
+                            //task.trace_timestamps("in_exec(subsequent list)");
+                        } else {
+                            found = false;
+                            break;
+                        }
                     }
-                    if task.already_finished() {
-                        task_cursor.remove();
+                    if should_remove && !removed {
+                        contended_unique_weights.remove_task(&uq);
                     }
-                    if let Some(new_cursor) = task_cursor.prev() {
-                        assert!(new_cursor.key() < task_cursor.key());
-                        task_cursor = new_cursor;
-                        task = task_cursor.value();
-                        //task.trace_timestamps("in_exec(subsequent list)");
-                    } else {
-                        found = false;
-                        break;
-                    }
-                }
-                if should_remove && !removed {
-                    contended_unique_weights.remove_task(&uq);
-                }
-                found.then(|| Task::clone_in_queue(task))
-            }).flatten().map(|task| {
-                //task.trace_timestamps(&format!("in_exec(heaviest:{})", self.task.queue_time_label()));
-                lock_attempt.heaviest_uncontended = Some(task);
-                ()
-            });
+                    found.then(|| Task::clone_in_queue(task))
+                })
+                .flatten()
+                .map(|task| {
+                    //task.trace_timestamps(&format!("in_exec(heaviest:{})", self.task.queue_time_label()));
+                    lock_attempt.heaviest_uncontended = Some(task);
+                    ()
+                });
         }
     }
 }
@@ -99,14 +107,14 @@ pub unsafe trait NotAtScheduleThread: Copy {}
 
 impl PageRc {
     fn page_mut<AST: AtScheduleThread>(&self, _ast: AST) -> std::cell::RefMut<'_, Page> {
-        self.0.0.borrow_mut()
+        self.0 .0.borrow_mut()
     }
 }
 
 #[derive(Clone, Debug)]
 enum LockStatus {
     Succeded,
-    Provisional, 
+    Provisional,
     Failed,
 }
 
@@ -142,7 +150,7 @@ impl LockAttempt {
     }
 
     pub fn contended_unique_weights(&self) -> &TaskIds {
-        &self.target.0.1
+        &self.target.0 .1
     }
 }
 
@@ -200,7 +208,9 @@ impl TaskIds {
     }
 
     #[inline(never)]
-    pub fn heaviest_task_cursor(&self) -> Option<crossbeam_skiplist::map::Entry<'_, UniqueWeight, TaskInQueue>> {
+    pub fn heaviest_task_cursor(
+        &self,
+    ) -> Option<crossbeam_skiplist::map::Entry<'_, UniqueWeight, TaskInQueue>> {
         self.task_ids.back()
     }
 }
@@ -258,7 +268,10 @@ struct ProvisioningTracker {
 
 impl ProvisioningTracker {
     fn new(remaining_count: usize, task: TaskInQueue) -> Self {
-        Self { remaining_count: std::sync::atomic::AtomicUsize::new(remaining_count), task }
+        Self {
+            remaining_count: std::sync::atomic::AtomicUsize::new(remaining_count),
+            task,
+        }
     }
 
     fn is_fulfilled(&self) -> bool {
@@ -266,7 +279,8 @@ impl ProvisioningTracker {
     }
 
     fn progress(&self) {
-        self.remaining_count.fetch_sub(1, std::sync::atomic::Ordering::SeqCst);
+        self.remaining_count
+            .fetch_sub(1, std::sync::atomic::Ordering::SeqCst);
     }
 
     fn prev_count(&self) -> usize {
@@ -274,7 +288,8 @@ impl ProvisioningTracker {
     }
 
     fn count(&self) -> usize {
-        self.remaining_count.load(std::sync::atomic::Ordering::SeqCst)
+        self.remaining_count
+            .load(std::sync::atomic::Ordering::SeqCst)
     }
 }
 
@@ -287,72 +302,80 @@ impl AddressBook {
         unique_weight: &UniqueWeight,
         attempt: &mut LockAttempt,
     ) -> CU {
-        let LockAttempt {target, requested_usage, status/*, remembered*/, ..} = attempt;
+        let LockAttempt {
+            target,
+            requested_usage,
+            status, /*, remembered*/
+            ..
+        } = attempt;
 
-                let mut page = target.page_mut(ast);
+        let mut page = target.page_mut(ast);
 
-                let next_usage = page.next_usage;
-                match page.current_usage {
-                    Usage::Unused => {
-                        assert_eq!(page.next_usage, Usage::Unused);
-                        page.current_usage = Usage::renew(*requested_usage);
-                        *status = LockStatus::Succeded;
+        let next_usage = page.next_usage;
+        match page.current_usage {
+            Usage::Unused => {
+                assert_eq!(page.next_usage, Usage::Unused);
+                page.current_usage = Usage::renew(*requested_usage);
+                *status = LockStatus::Succeded;
+            }
+            Usage::Readonly(ref mut count) => match requested_usage {
+                RequestedUsage::Readonly => {
+                    // prevent newer read-locks (even from runnable too)
+                    match next_usage {
+                        Usage::Unused => {
+                            *count += 1;
+                            *status = LockStatus::Succeded;
+                        }
+                        Usage::Readonly(_) | Usage::Writable => {
+                            *status = LockStatus::Failed;
+                        }
                     }
-                    Usage::Readonly(ref mut count) => match requested_usage {
-                        RequestedUsage::Readonly => {
-                            // prevent newer read-locks (even from runnable too)
-                            match next_usage {
-                                Usage::Unused => {
-                                    *count += 1;
-                                    *status = LockStatus::Succeded;
-                                },
-                                Usage::Readonly(_) | Usage::Writable => {
-                                    *status = LockStatus::Failed;
-                                }
+                }
+                RequestedUsage::Writable => {
+                    if from_runnable || prefer_immediate {
+                        *status = LockStatus::Failed;
+                    } else {
+                        match page.next_usage {
+                            Usage::Unused => {
+                                *status = LockStatus::Provisional;
+                                page.next_usage = Usage::renew(*requested_usage);
                             }
-                        }
-                        RequestedUsage::Writable => {
-                            if from_runnable || prefer_immediate {
+                            // support multiple readonly locks!
+                            Usage::Readonly(_) | Usage::Writable => {
                                 *status = LockStatus::Failed;
-                            } else {
-                                match page.next_usage {
-                                    Usage::Unused => {
-                                        *status = LockStatus::Provisional;
-                                        page.next_usage = Usage::renew(*requested_usage);
-                                    },
-                                    // support multiple readonly locks!
-                                    Usage::Readonly(_) | Usage::Writable => {
-                                        *status = LockStatus::Failed;
-                                    },
-                                }
                             }
                         }
-                    },
-                    Usage::Writable => {
-                        if from_runnable || prefer_immediate {
+                    }
+                }
+            },
+            Usage::Writable => {
+                if from_runnable || prefer_immediate {
+                    *status = LockStatus::Failed;
+                } else {
+                    match page.next_usage {
+                        Usage::Unused => {
+                            *status = LockStatus::Provisional;
+                            page.next_usage = Usage::renew(*requested_usage);
+                        }
+                        // support multiple readonly locks!
+                        Usage::Readonly(_) | Usage::Writable => {
                             *status = LockStatus::Failed;
-                        } else {
-                            match page.next_usage {
-                                Usage::Unused => {
-                                    *status = LockStatus::Provisional;
-                                    page.next_usage = Usage::renew(*requested_usage);
-                                },
-                                // support multiple readonly locks!
-                                Usage::Readonly(_) | Usage::Writable => {
-                                    *status = LockStatus::Failed;
-                                },
-                            }
                         }
-                    },
+                    }
                 }
+            }
+        }
         page.cu
     }
 
-    fn reset_lock<AST: AtScheduleThread>(&mut self, ast: AST, attempt: &mut LockAttempt, after_execution: bool) -> bool {
+    fn reset_lock<AST: AtScheduleThread>(
+        &mut self,
+        ast: AST,
+        attempt: &mut LockAttempt,
+        after_execution: bool,
+    ) -> bool {
         match attempt.status {
-            LockStatus::Succeded => {
-                self.unlock(ast, attempt)
-            },
+            LockStatus::Succeded => self.unlock(ast, attempt),
             LockStatus::Provisional => {
                 if after_execution {
                     self.unlock(ast, attempt)
@@ -409,16 +432,18 @@ impl AddressBook {
         match page.next_usage {
             Usage::Unused => {
                 unreachable!();
-            },
+            }
             // support multiple readonly locks!
             Usage::Readonly(_) | Usage::Writable => {
                 page.next_usage = Usage::Unused;
-            },
+            }
         }
     }
 
     pub fn preloader(&self) -> Preloader {
-        Preloader{book: std::sync::Arc::clone(&self.book)}
+        Preloader {
+            book: std::sync::Arc::clone(&self.book),
+        }
     }
 }
 
@@ -429,7 +454,12 @@ pub struct Preloader {
 impl Preloader {
     #[inline(never)]
     pub fn load(&self, address: Pubkey) -> PageRc {
-        PageRc::clone(&self.book.entry(address).or_insert_with(|| PageRc(PageRcInner::new((core::cell::RefCell::new(Page::new(Usage::unused())), Default::default())))))
+        PageRc::clone(&self.book.entry(address).or_insert_with(|| {
+            PageRc(PageRcInner::new((
+                core::cell::RefCell::new(Page::new(Usage::unused())),
+                Default::default(),
+            )))
+        }))
     }
 }
 
@@ -493,9 +523,15 @@ impl LockAttemptsInCell {
 // commit_time  -+
 
 impl Task {
-    pub fn new_for_queue<NAST: NotAtScheduleThread>(nast: NAST, unique_weight: UniqueWeight, tx: (SanitizedTransaction, Vec<LockAttempt>)) -> TaskInQueue {
+    pub fn new_for_queue<NAST: NotAtScheduleThread>(
+        nast: NAST,
+        unique_weight: UniqueWeight,
+        tx: (SanitizedTransaction, Vec<LockAttempt>),
+    ) -> TaskInQueue {
         TaskInQueue::new(Self {
-            for_indexer: LockAttemptsInCell::new(std::cell::RefCell::new(tx.1.iter().map(|a| a.clone_for_test()).collect())),
+            for_indexer: LockAttemptsInCell::new(std::cell::RefCell::new(
+                tx.1.iter().map(|a| a.clone_for_test()).collect(),
+            )),
             unique_weight,
             tx: (tx.0, LockAttemptsInCell::new(std::cell::RefCell::new(tx.1))),
             contention_count: Default::default(),
@@ -506,7 +542,7 @@ impl Task {
             queue_time: std::sync::atomic::AtomicUsize::new(usize::max_value()),
             queue_end_time: std::sync::atomic::AtomicUsize::new(usize::max_value()),
             execute_time: std::sync::atomic::AtomicUsize::new(usize::max_value()),
-            commit_time: std::sync::atomic::AtomicUsize::new(usize::max_value())
+            commit_time: std::sync::atomic::AtomicUsize::new(usize::max_value()),
         })
     }
 
@@ -515,16 +551,23 @@ impl Task {
         TaskInQueue::clone(this)
     }
 
-    fn lock_attempts_mut<AST: AtScheduleThread>(&self, _ast: AST) -> std::cell::RefMut<'_, Vec<LockAttempt>> {
-        self.tx.1.0.borrow_mut()
+    fn lock_attempts_mut<AST: AtScheduleThread>(
+        &self,
+        _ast: AST,
+    ) -> std::cell::RefMut<'_, Vec<LockAttempt>> {
+        self.tx.1 .0.borrow_mut()
     }
 
-    fn lock_attempts_not_mut<NAST: NotAtScheduleThread>(&self, _nast: NAST) -> std::cell::Ref<'_, Vec<LockAttempt>> {
-        self.tx.1.0.borrow()
+    fn lock_attempts_not_mut<NAST: NotAtScheduleThread>(
+        &self,
+        _nast: NAST,
+    ) -> std::cell::Ref<'_, Vec<LockAttempt>> {
+        self.tx.1 .0.borrow()
     }
 
     fn update_busiest_page_cu(&self, cu: CU) {
-        self.busiest_page_cu.store(cu, std::sync::atomic::Ordering::SeqCst);
+        self.busiest_page_cu
+            .store(cu, std::sync::atomic::Ordering::SeqCst);
     }
 
     pub fn record_sequence_time(&self, clock: usize) {
@@ -536,7 +579,8 @@ impl Task {
     }
 
     pub fn sequence_end_time(&self) -> usize {
-        self.sequence_end_time.load(std::sync::atomic::Ordering::SeqCst)
+        self.sequence_end_time
+            .load(std::sync::atomic::Ordering::SeqCst)
     }
 
     pub fn record_queue_time(&self, seq_clock: usize, queue_clock: usize) {
@@ -549,7 +593,8 @@ impl Task {
     }
 
     pub fn queue_end_time(&self) -> usize {
-        self.queue_end_time.load(std::sync::atomic::Ordering::SeqCst)
+        self.queue_end_time
+            .load(std::sync::atomic::Ordering::SeqCst)
     }
 
     pub fn record_execute_time(&self, queue_clock: usize, execute_clock: usize) {
@@ -570,9 +615,12 @@ impl Task {
     }
 
     pub fn queue_time_label(&self) -> String {
-        format!("queue: [{}qT..{}qT; {}qD]", 
-              self.queue_time(), self.queue_end_time(), self.queue_end_time() - self.queue_time(), 
-              )
+        format!(
+            "queue: [{}qT..{}qT; {}qD]",
+            self.queue_time(),
+            self.queue_end_time(),
+            self.queue_end_time() - self.queue_time(),
+        )
     }
 
     pub fn trace_timestamps(&self, prefix: &str) {
@@ -582,16 +630,28 @@ impl Task {
               self.sequence_time(),
               self.sequence_end_time(),
               self.sequence_end_time() - self.sequence_time(),
-              self.queue_time(), self.queue_end_time(), self.queue_end_time() - self.queue_time(), 
+              self.queue_time(), self.queue_end_time(), self.queue_end_time() - self.queue_time(),
               self.execute_time(), self.commit_time(), self.commit_time() - self.execute_time());
     }
 
-
     pub fn clone_for_test<NAST: NotAtScheduleThread>(&self, nast: NAST) -> Self {
         Self {
             unique_weight: self.unique_weight,
-            for_indexer: LockAttemptsInCell::new(std::cell::RefCell::new(self.lock_attempts_not_mut(nast).iter().map(|a| a.clone_for_test()).collect())),
-            tx: (self.tx.0.clone(), LockAttemptsInCell::new(std::cell::RefCell::new(self.lock_attempts_not_mut(nast).iter().map(|l| l.clone_for_test()).collect::<Vec<_>>()))),
+            for_indexer: LockAttemptsInCell::new(std::cell::RefCell::new(
+                self.lock_attempts_not_mut(nast)
+                    .iter()
+                    .map(|a| a.clone_for_test())
+                    .collect(),
+            )),
+            tx: (
+                self.tx.0.clone(),
+                LockAttemptsInCell::new(std::cell::RefCell::new(
+                    self.lock_attempts_not_mut(nast)
+                        .iter()
+                        .map(|l| l.clone_for_test())
+                        .collect::<Vec<_>>(),
+                )),
+            ),
             contention_count: Default::default(),
             busiest_page_cu: Default::default(),
             uncontended: Default::default(),
@@ -613,30 +673,40 @@ impl Task {
     }
 
     fn mark_as_contended(&self) {
-        self.uncontended.store(1, std::sync::atomic::Ordering::SeqCst)
+        self.uncontended
+            .store(1, std::sync::atomic::Ordering::SeqCst)
     }
 
     fn mark_as_uncontended(&self) {
         assert!(self.currently_contended());
-        self.uncontended.store(2, std::sync::atomic::Ordering::SeqCst)
+        self.uncontended
+            .store(2, std::sync::atomic::Ordering::SeqCst)
     }
 
     fn mark_as_finished(&self) {
         assert!(!self.already_finished() && !self.currently_contended());
-        self.uncontended.store(3, std::sync::atomic::Ordering::SeqCst)
+        self.uncontended
+            .store(3, std::sync::atomic::Ordering::SeqCst)
     }
 
     #[inline(never)]
-    fn index_to_address_book(this: &TaskInQueue, task_sender: &crossbeam_channel::Sender<(TaskInQueue, Vec<LockAttempt>)>) {
+    fn index_to_address_book(
+        this: &TaskInQueue,
+        task_sender: &crossbeam_channel::Sender<(TaskInQueue, Vec<LockAttempt>)>,
+    ) {
         //for lock_attempt in self.lock_attempts_mut(ast).iter() {
         //    lock_attempt.contended_unique_weights().insert_task(unique_weight, Task::clone_in_queue(&self));
         //}
         let a = Task::clone_in_queue(this);
-        task_sender.send((a, std::mem::take(&mut *this.for_indexer.0.borrow_mut()))).unwrap();
+        task_sender
+            .send((a, std::mem::take(&mut *this.for_indexer.0.borrow_mut())))
+            .unwrap();
     }
 
     fn stuck_task_id(&self) -> StuckTaskId {
-        let cu = self.busiest_page_cu.load(std::sync::atomic::Ordering::SeqCst);
+        let cu = self
+            .busiest_page_cu
+            .load(std::sync::atomic::Ordering::SeqCst);
         assert_ne!(cu, 0);
         (cu, TaskId::max_value() - self.unique_weight)
     }
@@ -660,7 +730,8 @@ pub type TaskInQueue = triomphe::Arc<Task>;
 //type TaskQueueEntry<'a> = dashmap::mapref::entry::Entry<'a, UniqueWeight, TaskInQueue>;
 //type TaskQueueOccupiedEntry<'a> = dashmap::mapref::entry::OccupiedEntry<'a, UniqueWeight, TaskInQueue, std::collections::hash_map::RandomState>;
 type TaskQueueEntry<'a> = std::collections::btree_map::Entry<'a, UniqueWeight, TaskInQueue>;
-type TaskQueueOccupiedEntry<'a> = std::collections::btree_map::OccupiedEntry<'a, UniqueWeight, TaskInQueue>;
+type TaskQueueOccupiedEntry<'a> =
+    std::collections::btree_map::OccupiedEntry<'a, UniqueWeight, TaskInQueue>;
 
 impl TaskQueue {
     #[inline(never)]
@@ -671,9 +742,7 @@ impl TaskQueue {
     }
 
     #[inline(never)]
-    fn heaviest_entry_to_execute(
-        &mut self,
-    ) -> Option<TaskQueueOccupiedEntry<'_>> {
+    fn heaviest_entry_to_execute(&mut self) -> Option<TaskQueueOccupiedEntry<'_>> {
         self.tasks.last_entry()
     }
 
@@ -698,17 +767,23 @@ fn attempt_lock_for_execution<'a, AST: AtScheduleThread>(
     let mut busiest_page_cu = 1;
 
     for attempt in placeholder_attempts.iter_mut() {
-        let cu = AddressBook::attempt_lock_address(ast, from_runnable, prefer_immediate, unique_weight, attempt);
+        let cu = AddressBook::attempt_lock_address(
+            ast,
+            from_runnable,
+            prefer_immediate,
+            unique_weight,
+            attempt,
+        );
         busiest_page_cu = busiest_page_cu.max(cu);
 
         match attempt.status {
-            LockStatus::Succeded => {},
+            LockStatus::Succeded => {}
             LockStatus::Failed => {
                 unlockable_count += 1;
-            },
+            }
             LockStatus::Provisional => {
                 provisional_count += 1;
-            },
+            }
         }
     }
 
@@ -718,32 +793,29 @@ fn attempt_lock_for_execution<'a, AST: AtScheduleThread>(
 type PreprocessedTransaction = (SanitizedTransaction, Vec<LockAttempt>);
 
 pub fn get_transaction_priority_details(tx: &SanitizedTransaction) -> u64 {
-        use solana_program_runtime::compute_budget::ComputeBudget;
-        let mut compute_budget = ComputeBudget::default();
-        compute_budget
-            .process_instructions(
-                tx.message().program_instructions_iter(),
-                true, // use default units per instruction
-                true, // don't reject txs that use set compute unit price ix
-            )
-            .map(|d| d.get_priority()).unwrap_or_default()
+    use solana_program_runtime::compute_budget::ComputeBudget;
+    let mut compute_budget = ComputeBudget::default();
+    compute_budget
+        .process_instructions(
+            tx.message().program_instructions_iter(),
+            true, // use default units per instruction
+            true, // don't reject txs that use set compute unit price ix
+        )
+        .map(|d| d.get_priority())
+        .unwrap_or_default()
 }
 
 pub struct ScheduleStage {}
 
 impl ScheduleStage {
-    fn push_to_runnable_queue(
-        task: TaskInQueue,
-        runnable_queue: &mut TaskQueue,
-    ) {
-        runnable_queue.add_to_schedule(
-            task.unique_weight,
-            task,
-        );
+    fn push_to_runnable_queue(task: TaskInQueue, runnable_queue: &mut TaskQueue) {
+        runnable_queue.add_to_schedule(task.unique_weight, task);
     }
 
     #[inline(never)]
-    fn get_heaviest_from_contended<'a>(address_book: &'a mut AddressBook) -> Option<std::collections::btree_map::OccupiedEntry<'a, UniqueWeight, TaskInQueue>> {
+    fn get_heaviest_from_contended<'a>(
+        address_book: &'a mut AddressBook,
+    ) -> Option<std::collections::btree_map::OccupiedEntry<'a, UniqueWeight, TaskInQueue>> {
         address_book.uncontended_task_ids.last_entry()
     }
 
@@ -752,10 +824,7 @@ impl ScheduleStage {
         runnable_queue: &'a mut TaskQueue,
         address_book: &mut AddressBook,
         contended_count: &usize,
-    ) -> Option<(
-        bool,
-        TaskInQueue,
-    )> {
+    ) -> Option<(bool, TaskInQueue)> {
         match (
             runnable_queue.heaviest_entry_to_execute(),
             Self::get_heaviest_from_contended(address_book),
@@ -768,8 +837,8 @@ impl ScheduleStage {
             (None, Some(weight_from_contended)) => {
                 trace!("select: contended only");
                 let t = weight_from_contended.remove();
-                Some(( false, t))
-            },
+                Some((false, t))
+            }
             (Some(heaviest_runnable_entry), Some(weight_from_contended)) => {
                 let weight_from_runnable = heaviest_runnable_entry.key();
                 let uw = weight_from_contended.key();
@@ -781,7 +850,7 @@ impl ScheduleStage {
                 } else if uw > weight_from_runnable {
                     trace!("select: contended > runnnable");
                     let t = weight_from_contended.remove();
-                    Some(( false, t))
+                    Some((false, t))
                 } else {
                     unreachable!(
                         "identical unique weights shouldn't exist in both runnable and contended"
@@ -790,7 +859,8 @@ impl ScheduleStage {
             }
             (None, None) => {
                 trace!("select: none");
-                if runnable_queue.task_count() == 0 && /* *contended_count > 0 &&*/ address_book.stuck_tasks.len() > 0 {
+                if runnable_queue.task_count() == 0 && /* *contended_count > 0 &&*/ address_book.stuck_tasks.len() > 0
+                {
                     trace!("handling stuck...");
                     let (stuck_task_id, task) = address_book.stuck_tasks.pop_first().unwrap();
                     // ensure proper rekeying
@@ -822,7 +892,10 @@ impl ScheduleStage {
         provisioning_tracker_count: &mut usize,
     ) -> Option<(UniqueWeight, TaskInQueue, Vec<LockAttempt>)> {
         if let Some(mut a) = address_book.fulfilled_provisional_task_ids.pop_last() {
-            trace!("expediate pop from provisional queue [rest: {}]", address_book.fulfilled_provisional_task_ids.len());
+            trace!(
+                "expediate pop from provisional queue [rest: {}]",
+                address_book.fulfilled_provisional_task_ids.len()
+            );
 
             let lock_attempts = std::mem::take(&mut *a.1.lock_attempts_mut(ast));
 
@@ -831,104 +904,140 @@ impl ScheduleStage {
 
         trace!("pop begin");
         loop {
-        if let Some((from_runnable, mut next_task)) = Self::select_next_task(runnable_queue, address_book, contended_count) {
-            trace!("pop loop iteration");
-            if from_runnable {
-                next_task.record_queue_time(*sequence_clock, *queue_clock);
-                *queue_clock = queue_clock.checked_add(1).unwrap();
-            }
-            let unique_weight = next_task.unique_weight;
-            let message_hash = next_task.tx.0.message_hash();
-
-            // plumb message_hash into StatusCache or implmenent our own for duplicate tx
-            // detection?
-
-            let (unlockable_count, provisional_count, busiest_page_cu) = attempt_lock_for_execution(
-                ast,
-                from_runnable,
-                prefer_immediate,
-                address_book,
-                &unique_weight,
-                &message_hash,
-                &mut next_task.lock_attempts_mut(ast),
-            );
-
-            if unlockable_count > 0 {
-                //trace!("reset_lock_for_failed_execution(): {:?} {}", (&unique_weight, from_runnable), next_task.tx.0.signature());
-                Self::reset_lock_for_failed_execution(
-                    ast,
-                    address_book,
-                    &unique_weight,
-                    &mut next_task.lock_attempts_mut(ast),
-                    from_runnable,
-                );
-                let lock_count = next_task.lock_attempts_mut(ast).len();
-                next_task.contention_count.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
-
+            if let Some((from_runnable, mut next_task)) =
+                Self::select_next_task(runnable_queue, address_book, contended_count)
+            {
+                trace!("pop loop iteration");
                 if from_runnable {
-                    trace!("move to contended due to lock failure [{}/{}/{}]", unlockable_count, provisional_count, lock_count);
-                    next_task.mark_as_contended();
-                    *contended_count = contended_count.checked_add(1).unwrap();
-
-                    Task::index_to_address_book(&next_task, task_sender);
-
-                    // maybe run lightweight prune logic on contended_queue here.
-                } else {
-                    trace!("relock failed [{}/{}/{}]; remains in contended: {:?} contention: {}", unlockable_count, provisional_count, lock_count, &unique_weight, next_task.contention_count.load(std::sync::atomic::Ordering::SeqCst));
-                    //address_book.uncontended_task_ids.clear();
+                    next_task.record_queue_time(*sequence_clock, *queue_clock);
+                    *queue_clock = queue_clock.checked_add(1).unwrap();
                 }
+                let unique_weight = next_task.unique_weight;
+                let message_hash = next_task.tx.0.message_hash();
+
+                // plumb message_hash into StatusCache or implmenent our own for duplicate tx
+                // detection?
+
+                let (unlockable_count, provisional_count, busiest_page_cu) =
+                    attempt_lock_for_execution(
+                        ast,
+                        from_runnable,
+                        prefer_immediate,
+                        address_book,
+                        &unique_weight,
+                        &message_hash,
+                        &mut next_task.lock_attempts_mut(ast),
+                    );
+
+                if unlockable_count > 0 {
+                    //trace!("reset_lock_for_failed_execution(): {:?} {}", (&unique_weight, from_runnable), next_task.tx.0.signature());
+                    Self::reset_lock_for_failed_execution(
+                        ast,
+                        address_book,
+                        &unique_weight,
+                        &mut next_task.lock_attempts_mut(ast),
+                        from_runnable,
+                    );
+                    let lock_count = next_task.lock_attempts_mut(ast).len();
+                    next_task
+                        .contention_count
+                        .fetch_add(1, std::sync::atomic::Ordering::SeqCst);
+
+                    if from_runnable {
+                        trace!(
+                            "move to contended due to lock failure [{}/{}/{}]",
+                            unlockable_count,
+                            provisional_count,
+                            lock_count
+                        );
+                        next_task.mark_as_contended();
+                        *contended_count = contended_count.checked_add(1).unwrap();
+
+                        Task::index_to_address_book(&next_task, task_sender);
+
+                        // maybe run lightweight prune logic on contended_queue here.
+                    } else {
+                        trace!(
+                            "relock failed [{}/{}/{}]; remains in contended: {:?} contention: {}",
+                            unlockable_count,
+                            provisional_count,
+                            lock_count,
+                            &unique_weight,
+                            next_task
+                                .contention_count
+                                .load(std::sync::atomic::Ordering::SeqCst)
+                        );
+                        //address_book.uncontended_task_ids.clear();
+                    }
 
-                if from_runnable {
-                    next_task.update_busiest_page_cu(busiest_page_cu);
-                    let a = address_book.stuck_tasks.insert(next_task.stuck_task_id(), Task::clone_in_queue(&next_task));
-                    assert!(a.is_none());
-                    continue;
-                } else {
-                    // todo: remove this task from stuck_tasks before update_busiest_page_cu
-                    let removed = address_book.stuck_tasks.remove(&next_task.stuck_task_id()).unwrap();
+                    if from_runnable {
+                        next_task.update_busiest_page_cu(busiest_page_cu);
+                        let a = address_book
+                            .stuck_tasks
+                            .insert(next_task.stuck_task_id(), Task::clone_in_queue(&next_task));
+                        assert!(a.is_none());
+                        continue;
+                    } else {
+                        // todo: remove this task from stuck_tasks before update_busiest_page_cu
+                        let removed = address_book
+                            .stuck_tasks
+                            .remove(&next_task.stuck_task_id())
+                            .unwrap();
+                        next_task.update_busiest_page_cu(busiest_page_cu);
+                        let a = address_book
+                            .stuck_tasks
+                            .insert(next_task.stuck_task_id(), removed);
+                        assert!(a.is_none());
+                        return None;
+                    }
+                } else if provisional_count > 0 {
+                    assert!(!from_runnable);
+                    assert_eq!(unlockable_count, 0);
+                    let lock_count = next_task.lock_attempts_mut(ast).len();
+                    trace!("provisional exec: [{}/{}]", provisional_count, lock_count);
+                    *contended_count = contended_count.checked_sub(1).unwrap();
+                    next_task.mark_as_uncontended();
+                    address_book.stuck_tasks.remove(&next_task.stuck_task_id());
                     next_task.update_busiest_page_cu(busiest_page_cu);
-                    let a = address_book.stuck_tasks.insert(next_task.stuck_task_id(), removed);
-                    assert!(a.is_none());
+
+                    let tracker = triomphe::Arc::new(ProvisioningTracker::new(
+                        provisional_count,
+                        Task::clone_in_queue(&next_task),
+                    ));
+                    *provisioning_tracker_count =
+                        provisioning_tracker_count.checked_add(1).unwrap();
+                    Self::finalize_lock_for_provisional_execution(
+                        ast,
+                        address_book,
+                        &next_task,
+                        tracker,
+                    );
+
                     return None;
+                    continue;
                 }
-            } else if provisional_count > 0 {
-                assert!(!from_runnable);
-                assert_eq!(unlockable_count, 0);
-                let lock_count = next_task.lock_attempts_mut(ast).len();
-                trace!("provisional exec: [{}/{}]", provisional_count, lock_count);
-                *contended_count = contended_count.checked_sub(1).unwrap();
-                next_task.mark_as_uncontended();
-                address_book.stuck_tasks.remove(&next_task.stuck_task_id());
-                next_task.update_busiest_page_cu(busiest_page_cu);
-
-                let tracker = triomphe::Arc::new(ProvisioningTracker::new(provisional_count, Task::clone_in_queue(&next_task)));
-                *provisioning_tracker_count = provisioning_tracker_count.checked_add(1).unwrap();
-                Self::finalize_lock_for_provisional_execution(
-                    ast,
-                    address_book,
-                    &next_task,
-                    tracker
-                );
 
-                return None;
-                continue;
-            }
+                trace!(
+                    "successful lock: (from_runnable: {}) after {} contentions",
+                    from_runnable,
+                    next_task
+                        .contention_count
+                        .load(std::sync::atomic::Ordering::SeqCst)
+                );
 
-            trace!("successful lock: (from_runnable: {}) after {} contentions", from_runnable, next_task.contention_count.load(std::sync::atomic::Ordering::SeqCst));
+                assert!(!next_task.already_finished());
+                if !from_runnable {
+                    *contended_count = contended_count.checked_sub(1).unwrap();
+                    next_task.mark_as_uncontended();
+                } else {
+                    next_task.update_busiest_page_cu(busiest_page_cu);
+                }
+                let lock_attempts = std::mem::take(&mut *next_task.lock_attempts_mut(ast));
 
-            assert!(!next_task.already_finished());
-            if !from_runnable {
-                *contended_count = contended_count.checked_sub(1).unwrap();
-                next_task.mark_as_uncontended();
+                return Some((unique_weight, next_task, lock_attempts));
             } else {
-                next_task.update_busiest_page_cu(busiest_page_cu);
+                break;
             }
-            let lock_attempts = std::mem::take(&mut *next_task.lock_attempts_mut(ast));
-
-            return Some((unique_weight, next_task, lock_attempts));
-        } else {
-            break;
-        }
         }
 
         None
@@ -944,7 +1053,10 @@ impl ScheduleStage {
         for l in next_task.lock_attempts_mut(ast).iter_mut() {
             match l.status {
                 LockStatus::Provisional => {
-                    l.target.page_mut(ast).provisional_task_ids.push(triomphe::Arc::clone(&tracker));
+                    l.target
+                        .page_mut(ast)
+                        .provisional_task_ids
+                        .push(triomphe::Arc::clone(&tracker));
                 }
                 LockStatus::Succeded => {
                     // do nothing
@@ -971,7 +1083,13 @@ impl ScheduleStage {
     }
 
     #[inline(never)]
-    fn unlock_after_execution<AST: AtScheduleThread>(ast: AST, address_book: &mut AddressBook, lock_attempts: &mut [LockAttempt], provisioning_tracker_count: &mut usize, cu: CU) {
+    fn unlock_after_execution<AST: AtScheduleThread>(
+        ast: AST,
+        address_book: &mut AddressBook,
+        lock_attempts: &mut [LockAttempt],
+        provisioning_tracker_count: &mut usize,
+        cu: CU,
+    ) {
         for mut l in lock_attempts {
             let newly_uncontended = address_book.reset_lock(ast, &mut l, true);
 
@@ -982,33 +1100,37 @@ impl ScheduleStage {
 
                 if let Some(task) = l.heaviest_uncontended.take() {
                     //assert!(!task.already_finished());
-                    if /*true ||*/ task.currently_contended() {
+                    if
+                    /*true ||*/
+                    task.currently_contended() {
                         //assert!(task.currently_contended());
                         //inserted = true;
-                        address_book.uncontended_task_ids.insert(task.unique_weight, task);
+                        address_book
+                            .uncontended_task_ids
+                            .insert(task.unique_weight, task);
                     } /*else {
-                        let contended_unique_weights = &page.contended_unique_weights;
-                        contended_unique_weights.heaviest_task_cursor().map(|mut task_cursor| {
-                            let mut found = true;
-                            //assert_ne!(task_cursor.key(), &task.uq);
-                            let mut task = task_cursor.value();
-                            while !task.currently_contended() {
-                                if let Some(new_cursor) = task_cursor.prev() {
-                                    assert!(new_cursor.key() < task_cursor.key());
-                                    //assert_ne!(new_cursor.key(), &uq);
-                                    task_cursor = new_cursor;
-                                    task = task_cursor.value();
-                                } else {
-                                    found = false;
-                                    break;
-                                }
-                            }
-                            found.then(|| Task::clone_in_queue(task))
-                        }).flatten().map(|task| {
-                            address_book.uncontended_task_ids.insert(task.unique_weight, task);
-                            ()
-                        });
-                    }*/
+                          let contended_unique_weights = &page.contended_unique_weights;
+                          contended_unique_weights.heaviest_task_cursor().map(|mut task_cursor| {
+                              let mut found = true;
+                              //assert_ne!(task_cursor.key(), &task.uq);
+                              let mut task = task_cursor.value();
+                              while !task.currently_contended() {
+                                  if let Some(new_cursor) = task_cursor.prev() {
+                                      assert!(new_cursor.key() < task_cursor.key());
+                                      //assert_ne!(new_cursor.key(), &uq);
+                                      task_cursor = new_cursor;
+                                      task = task_cursor.value();
+                                  } else {
+                                      found = false;
+                                      break;
+                                  }
+                              }
+                              found.then(|| Task::clone_in_queue(task))
+                          }).flatten().map(|task| {
+                              address_book.uncontended_task_ids.insert(task.unique_weight, task);
+                              ()
+                          });
+                      }*/
                 }
             }
             if page.current_usage == Usage::Unused && page.next_usage != Usage::Unused {
@@ -1016,11 +1138,23 @@ impl ScheduleStage {
                 for tracker in std::mem::take(&mut page.provisional_task_ids).into_iter() {
                     tracker.progress();
                     if tracker.is_fulfilled() {
-                        trace!("provisioning tracker progress: {} => {} (!)", tracker.prev_count(), tracker.count());
-                        address_book.fulfilled_provisional_task_ids.insert(tracker.task.unique_weight, Task::clone_in_queue(&tracker.task));
-                        *provisioning_tracker_count = provisioning_tracker_count.checked_sub(1).unwrap();
+                        trace!(
+                            "provisioning tracker progress: {} => {} (!)",
+                            tracker.prev_count(),
+                            tracker.count()
+                        );
+                        address_book.fulfilled_provisional_task_ids.insert(
+                            tracker.task.unique_weight,
+                            Task::clone_in_queue(&tracker.task),
+                        );
+                        *provisioning_tracker_count =
+                            provisioning_tracker_count.checked_sub(1).unwrap();
                     } else {
-                        trace!("provisioning tracker progress: {} => {}", tracker.prev_count(), tracker.count());
+                        trace!(
+                            "provisioning tracker progress: {} => {}",
+                            tracker.prev_count(),
+                            tracker.count()
+                        );
                     }
                 }
             }
@@ -1052,7 +1186,13 @@ impl ScheduleStage {
     }
 
     #[inline(never)]
-    fn commit_completed_execution<AST: AtScheduleThread>(ast: AST, ee: &mut ExecutionEnvironment, address_book: &mut AddressBook, commit_time: &mut usize, provisioning_tracker_count: &mut usize) {
+    fn commit_completed_execution<AST: AtScheduleThread>(
+        ast: AST,
+        ee: &mut ExecutionEnvironment,
+        address_book: &mut AddressBook,
+        commit_time: &mut usize,
+        provisioning_tracker_count: &mut usize,
+    ) {
         // do par()-ly?
 
         //ee.reindex();
@@ -1061,7 +1201,13 @@ impl ScheduleStage {
         //*commit_time = commit_time.checked_add(1).unwrap();
 
         // which order for data race free?: unlocking / marking
-        Self::unlock_after_execution(ast, address_book, &mut ee.finalized_lock_attempts, provisioning_tracker_count, ee.cu);
+        Self::unlock_after_execution(
+            ast,
+            address_book,
+            &mut ee.finalized_lock_attempts,
+            provisioning_tracker_count,
+            ee.cu,
+        );
         ee.task.mark_as_finished();
 
         address_book.stuck_tasks.remove(&ee.task.stuck_task_id());
@@ -1088,9 +1234,20 @@ impl ScheduleStage {
         execute_clock: &mut usize,
         provisioning_tracker_count: &mut usize,
     ) -> Option<Box<ExecutionEnvironment>> {
-        let maybe_ee =
-            Self::pop_from_queue_then_lock(ast, task_sender, runnable_queue, address_book, contended_count, prefer_immediate, sequence_time, queue_clock, provisioning_tracker_count)
-                .map(|(uw, t,ll)| Self::prepare_scheduled_execution(address_book, uw, t, ll, queue_clock, execute_clock));
+        let maybe_ee = Self::pop_from_queue_then_lock(
+            ast,
+            task_sender,
+            runnable_queue,
+            address_book,
+            contended_count,
+            prefer_immediate,
+            sequence_time,
+            queue_clock,
+            provisioning_tracker_count,
+        )
+        .map(|(uw, t, ll)| {
+            Self::prepare_scheduled_execution(address_book, uw, t, ll, queue_clock, execute_clock)
+        });
         maybe_ee
     }
 
@@ -1115,7 +1272,6 @@ impl ScheduleStage {
         to_execute_substage: &crossbeam_channel::Sender<Box<ExecutionEnvironment>>,
         maybe_to_next_stage: Option<&crossbeam_channel::Sender<Box<ExecutionEnvironment>>>, // assume nonblocking
     ) {
-
         let mut executing_queue_count = 0_usize;
         let mut contended_count = 0;
         let mut provisioning_tracker_count = 0;
@@ -1127,42 +1283,51 @@ impl ScheduleStage {
         let (to_next_stage, maybe_jon_handle) = if let Some(to_next_stage) = maybe_to_next_stage {
             (to_next_stage, None)
         } else {
-            let h = std::thread::Builder::new().name("sol-reaper".to_string()).spawn(move || {
-                #[derive(Clone, Copy, Debug)]
-                struct NotAtTopOfScheduleThread;
-                unsafe impl NotAtScheduleThread for NotAtTopOfScheduleThread {}
-                let nast = NotAtTopOfScheduleThread;
-
-                while let Ok(mut a) = ee_receiver.recv() {
-                    assert!(a.task.lock_attempts_not_mut(nast).is_empty());
-                    //assert!(a.task.sequence_time() != usize::max_value());
-                    //let lock_attempts = std::mem::take(&mut a.lock_attempts);
-                    //drop(lock_attempts);
-                    //TaskInQueue::get_mut(&mut a.task).unwrap();
-                }
-                assert_eq!(ee_receiver.len(), 0);
-            }).unwrap();
+            let h = std::thread::Builder::new()
+                .name("sol-reaper".to_string())
+                .spawn(move || {
+                    #[derive(Clone, Copy, Debug)]
+                    struct NotAtTopOfScheduleThread;
+                    unsafe impl NotAtScheduleThread for NotAtTopOfScheduleThread {}
+                    let nast = NotAtTopOfScheduleThread;
+
+                    while let Ok(mut a) = ee_receiver.recv() {
+                        assert!(a.task.lock_attempts_not_mut(nast).is_empty());
+                        //assert!(a.task.sequence_time() != usize::max_value());
+                        //let lock_attempts = std::mem::take(&mut a.lock_attempts);
+                        //drop(lock_attempts);
+                        //TaskInQueue::get_mut(&mut a.task).unwrap();
+                    }
+                    assert_eq!(ee_receiver.len(), 0);
+                })
+                .unwrap();
 
             (&ee_sender, Some(h))
         };
-        let (task_sender, task_receiver) = crossbeam_channel::unbounded::<(TaskInQueue, Vec<LockAttempt>)>();
+        let (task_sender, task_receiver) =
+            crossbeam_channel::unbounded::<(TaskInQueue, Vec<LockAttempt>)>();
         let indexer_count = std::env::var("INDEXER_COUNT")
             .unwrap_or(format!("{}", 4))
             .parse::<usize>()
             .unwrap();
         for thx in 0..indexer_count {
             let task_receiver = task_receiver.clone();
-            let h = std::thread::Builder::new().name(format!("sol-indexer{:02}", thx)).spawn(move || {
-                while let Ok((task, ll)) = task_receiver.recv() {
-                    for lock_attempt in ll {
-                        if task.already_finished() {
-                            break;
+            let h = std::thread::Builder::new()
+                .name(format!("sol-indexer{:02}", thx))
+                .spawn(move || {
+                    while let Ok((task, ll)) = task_receiver.recv() {
+                        for lock_attempt in ll {
+                            if task.already_finished() {
+                                break;
+                            }
+                            lock_attempt
+                                .contended_unique_weights()
+                                .insert_task(task.unique_weight, Task::clone_in_queue(&task));
                         }
-                        lock_attempt.contended_unique_weights().insert_task(task.unique_weight, Task::clone_in_queue(&task));
                     }
-                }
-                assert_eq!(task_receiver.len(), 0);
-            }).unwrap();
+                    assert_eq!(task_receiver.len(), 0);
+                })
+                .unwrap();
         }
         let mut start = std::time::Instant::now();
 
@@ -1204,14 +1369,27 @@ impl ScheduleStage {
             let (mut from_len, mut from_exec_len) = (0, 0);
 
             loop {
-                while (executing_queue_count + provisioning_tracker_count) < max_executing_queue_count {
+                while (executing_queue_count + provisioning_tracker_count)
+                    < max_executing_queue_count
+                {
                     trace!("schedule_once (from: {}, to: {}, runnnable: {}, contended: {}, (immediate+provisional)/max: ({}+{})/{}) active from contended: {} stuck: {}!", from.len(), to_execute_substage.len(), runnable_queue.task_count(), contended_count, executing_queue_count, provisioning_tracker_count, max_executing_queue_count, address_book.uncontended_task_ids.len(), address_book.stuck_tasks.len());
                     if start.elapsed() > std::time::Duration::from_millis(1000) {
                         start = std::time::Instant::now();
                         info!("schedule_once (from: {}, to: {}, runnnable: {}, contended: {}, (immediate+provisional)/max: ({}+{})/{}) active from contended: {} stuck: {}!", from.len(), to_execute_substage.len(), runnable_queue.task_count(), contended_count, executing_queue_count, provisioning_tracker_count, max_executing_queue_count, address_book.uncontended_task_ids.len(), address_book.stuck_tasks.len());
                     }
-                    let prefer_immediate = provisioning_tracker_count/4 > executing_queue_count;
-                    if let Some(ee) = Self::schedule_next_execution(ast, &task_sender, runnable_queue, address_book, &mut contended_count, prefer_immediate, &sequence_time, &mut queue_clock, &mut execute_clock, &mut provisioning_tracker_count) {
+                    let prefer_immediate = provisioning_tracker_count / 4 > executing_queue_count;
+                    if let Some(ee) = Self::schedule_next_execution(
+                        ast,
+                        &task_sender,
+                        runnable_queue,
+                        address_book,
+                        &mut contended_count,
+                        prefer_immediate,
+                        &sequence_time,
+                        &mut queue_clock,
+                        &mut execute_clock,
+                        &mut provisioning_tracker_count,
+                    ) {
                         executing_queue_count = executing_queue_count.checked_add(1).unwrap();
                         to_execute_substage.send(ee).unwrap();
                     } else {
@@ -1221,7 +1399,7 @@ impl ScheduleStage {
                 //break;
                 if first_iteration {
                     first_iteration = false;
-                    (from_len,  from_exec_len) = (from.len(), from_exec.len());
+                    (from_len, from_exec_len) = (from.len(), from_exec.len());
                 } else {
                     if empty_from {
                         from_len = from.len();
@@ -1233,21 +1411,27 @@ impl ScheduleStage {
                 (empty_from, empty_from_exec) = (from_len == 0, from_exec_len == 0);
 
                 if empty_from && empty_from_exec {
-                   break;
+                    break;
                 } else {
                     if !empty_from_exec {
                         let mut processed_execution_environment = from_exec.recv().unwrap();
                         from_exec_len = from_exec_len.checked_sub(1).unwrap();
                         empty_from_exec = from_exec_len == 0;
                         executing_queue_count = executing_queue_count.checked_sub(1).unwrap();
-                        Self::commit_completed_execution(ast, &mut processed_execution_environment, address_book, &mut execute_clock, &mut provisioning_tracker_count);
+                        Self::commit_completed_execution(
+                            ast,
+                            &mut processed_execution_environment,
+                            address_book,
+                            &mut execute_clock,
+                            &mut provisioning_tracker_count,
+                        );
                         to_next_stage.send(processed_execution_environment).unwrap();
                     }
                     if !empty_from {
-                       let task = from.recv().unwrap();
-                       from_len = from_len.checked_sub(1).unwrap();
-                       empty_from = from_len == 0;
-                       Self::register_runnable_task(task, runnable_queue, &mut sequence_time);
+                        let task = from.recv().unwrap();
+                        from_len = from_len.checked_sub(1).unwrap();
+                        empty_from = from_len == 0;
+                        Self::register_runnable_task(task, runnable_queue, &mut sequence_time);
                     }
                 }
             }
@@ -1269,7 +1453,16 @@ impl ScheduleStage {
         struct AtTopOfScheduleThread;
         unsafe impl AtScheduleThread for AtTopOfScheduleThread {}
 
-        Self::_run::<AtTopOfScheduleThread>(AtTopOfScheduleThread, max_executing_queue_count, runnable_queue, address_book, from, from_exec, to_execute_substage, maybe_to_next_stage)
+        Self::_run::<AtTopOfScheduleThread>(
+            AtTopOfScheduleThread,
+            max_executing_queue_count,
+            runnable_queue,
+            address_book,
+            from,
+            from_exec,
+            to_execute_substage,
+            maybe_to_next_stage,
+        )
     }
 }
 

From d0de377304cab198faa7916d0bf9a52a436406e5 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 3 Sep 2022 14:41:18 +0900
Subject: [PATCH 1853/3199] save

---
 scheduler/src/lib.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 1a2a14dea75801..5fdb3721138f88 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -759,14 +759,14 @@ fn attempt_lock_for_execution<'a, AST: AtScheduleThread>(
     address_book: &mut AddressBook,
     unique_weight: &UniqueWeight,
     message_hash: &'a Hash,
-    placeholder_attempts: &mut Vec<LockAttempt>,
+    lock_attempts: &mut [LockAttempt],
 ) -> (usize, usize, CU) {
     // no short-cuircuit; we at least all need to add to the contended queue
     let mut unlockable_count = 0;
     let mut provisional_count = 0;
     let mut busiest_page_cu = 1;
 
-    for attempt in placeholder_attempts.iter_mut() {
+    for attempt in lock_attempts.iter_mut() {
         let cu = AddressBook::attempt_lock_address(
             ast,
             from_runnable,

From 72fe7738a8d961416de60e76dd973210339808ac Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 3 Sep 2022 14:49:32 +0900
Subject: [PATCH 1854/3199] Rever some for merge

---
 runtime/src/vote_parser.rs        | 2 +-
 sdk/program/src/message/legacy.rs | 5 ++---
 sdk/src/transaction/sanitized.rs  | 2 ++
 3 files changed, 5 insertions(+), 4 deletions(-)

diff --git a/runtime/src/vote_parser.rs b/runtime/src/vote_parser.rs
index e761772982b6c7..c67045b0827533 100644
--- a/runtime/src/vote_parser.rs
+++ b/runtime/src/vote_parser.rs
@@ -13,7 +13,7 @@ use {
 pub type ParsedVote = (Pubkey, VoteTransaction, Option<Hash>, Signature);
 
 // Used for filtering out votes from the transaction log collector
-pub fn is_simple_vote_transaction(transaction: &SanitizedTransaction) -> bool {
+pub(crate) fn is_simple_vote_transaction(transaction: &SanitizedTransaction) -> bool {
     if transaction.message().instructions().len() == 1 {
         let (program_pubkey, instruction) = transaction
             .message()
diff --git a/sdk/program/src/message/legacy.rs b/sdk/program/src/message/legacy.rs
index 1faea0e64caaab..8b90322400253f 100644
--- a/sdk/program/src/message/legacy.rs
+++ b/sdk/program/src/message/legacy.rs
@@ -467,9 +467,8 @@ impl Message {
     /// Compute the blake3 hash of this transaction's message.
     #[cfg(not(target_os = "solana"))]
     pub fn hash(&self) -> Hash {
-        //let message_bytes = self.serialize();
-        //Self::hash_raw_message(&message_bytes)
-        Hash::default()
+        let message_bytes = self.serialize();
+        Self::hash_raw_message(&message_bytes)
     }
 
     /// Compute the blake3 hash of a raw transaction message.
diff --git a/sdk/src/transaction/sanitized.rs b/sdk/src/transaction/sanitized.rs
index 46c095703c83bf..35a379f724da0a 100644
--- a/sdk/src/transaction/sanitized.rs
+++ b/sdk/src/transaction/sanitized.rs
@@ -145,6 +145,8 @@ impl SanitizedTransaction {
     }
 
     pub fn try_from_legacy_transaction(tx: Transaction) -> Result<Self> {
+        tx.sanitize()?;
+
         Ok(Self {
             message_hash: tx.message.hash(),
             message: SanitizedMessage::Legacy(tx.message),

From 4a2a203ac7cae05ff88cbffcccbdb9020f63d16f Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 3 Sep 2022 16:56:48 +0900
Subject: [PATCH 1855/3199] save

---
 core/src/replay_stage.rs |  1 +
 runtime/src/bank.rs      | 24 ++++++++++++++++++++++++
 2 files changed, 25 insertions(+)

diff --git a/core/src/replay_stage.rs b/core/src/replay_stage.rs
index 44f2ef5c727d52..dfe5674c17f2b8 100644
--- a/core/src/replay_stage.rs
+++ b/core/src/replay_stage.rs
@@ -2472,6 +2472,7 @@ impl ReplayStage {
 
             assert_eq!(bank_slot, bank.slot());
             if bank.is_complete() {
+                bank.wait_for_scheduler();
                 let mut bank_complete_time = Measure::start("bank_complete_time");
                 let bank_progress = progress
                     .get_mut(&bank.slot())
diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 4d008186e2a750..64ff085bf3ce5e 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1114,6 +1114,7 @@ impl PartialEq for Bank {
             accounts_data_size_delta_off_chain: _,
             fee_structure: _,
             incremental_snapshot_persistence: _,
+            scheduler: _,
             // Ignore new fields explicitly if they do not impact PartialEq.
             // Adding ".." will remove compile-time checks that if a new field
             // is added to the struct, this ParitalEq is accordingly updated.
@@ -1195,6 +1196,22 @@ impl AbiExample for BuiltinPrograms {
     }
 }
 
+#[derive(Debug)]
+struct Scheduler {
+    scheduler_thread_handle: std::thread::JoinHandle<()>,
+}
+
+impl Default for Scheduler {
+    fn default() -> Self {
+        let scheduler_thread_handle = std::thread::Builder::new().name("sol-scheduler".to_string()).spawn(move || {
+        }).unwrap();
+
+        Self {
+            scheduler_thread_handle
+        }
+    }
+}
+
 /// Manager for the state of all accounts and programs after processing its entries.
 /// AbiExample is needed even without Serialize/Deserialize; actual (de-)serialization
 /// are implemented elsewhere for versioning
@@ -1369,6 +1386,8 @@ pub struct Bank {
     pub fee_structure: FeeStructure,
 
     pub incremental_snapshot_persistence: Option<BankIncrementalSnapshotPersistence>,
+
+    scheduler: Scheduler,
 }
 
 struct VoteWithStakeDelegations {
@@ -1559,6 +1578,7 @@ impl Bank {
             accounts_data_size_delta_on_chain: AtomicI64::new(0),
             accounts_data_size_delta_off_chain: AtomicI64::new(0),
             fee_structure: FeeStructure::default(),
+            scheduler: Scheduler::default(),
         };
 
         let accounts_data_size_initial = bank.get_total_accounts_stats().unwrap().data_len as u64;
@@ -1798,6 +1818,8 @@ impl Bank {
             measure!(parent.feature_set.clone(), "feature_set_creation");
 
         let accounts_data_size_initial = parent.load_accounts_data_size();
+        let scheduler = Scheduler::default();
+
         let mut new = Bank {
             incremental_snapshot_persistence: None,
             rewrites_skipped_this_slot: Rewrites::default(),
@@ -1877,6 +1899,7 @@ impl Bank {
             accounts_data_size_delta_on_chain: AtomicI64::new(0),
             accounts_data_size_delta_off_chain: AtomicI64::new(0),
             fee_structure: parent.fee_structure.clone(),
+            scheduler
         };
 
         let (_, ancestors_time) = measure!(
@@ -2221,6 +2244,7 @@ impl Bank {
             accounts_data_size_delta_on_chain: AtomicI64::new(0),
             accounts_data_size_delta_off_chain: AtomicI64::new(0),
             fee_structure: FeeStructure::default(),
+            scheduler: Scheduler::default(),
         };
         bank.finish_init(
             genesis_config,

From 1853cd48c839be9b0cbbb42a09a2073825f3894a Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 3 Sep 2022 17:03:06 +0900
Subject: [PATCH 1856/3199] save

---
 runtime/src/bank.rs | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 64ff085bf3ce5e..31b738775ad2de 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -7867,6 +7867,10 @@ impl Bank {
 
         total_accounts_stats
     }
+
+    fn wait_for_scheduler(&self) {
+        self.scheduler.scheduler_thread_handle.join()
+    }
 }
 
 /// Compute how much an account has changed size.  This function is useful when the data size delta

From e7df46f3f87a5b18cec66242133152c5e516315f Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 3 Sep 2022 17:03:19 +0900
Subject: [PATCH 1857/3199] save

---
 runtime/src/bank.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 31b738775ad2de..1f2d24d326a0a8 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -7869,7 +7869,7 @@ impl Bank {
     }
 
     fn wait_for_scheduler(&self) {
-        self.scheduler.scheduler_thread_handle.join()
+        self.scheduler.scheduler_thread_handle.join();
     }
 }
 

From 446f7da885a356e168ebeea1a3f68c0c1b187b54 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 3 Sep 2022 17:03:56 +0900
Subject: [PATCH 1858/3199] save

---
 runtime/src/bank.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 1f2d24d326a0a8..031485f0ab1622 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1387,7 +1387,7 @@ pub struct Bank {
 
     pub incremental_snapshot_persistence: Option<BankIncrementalSnapshotPersistence>,
 
-    scheduler: Scheduler,
+    scheduler: ArcSwap<Scheduler>,
 }
 
 struct VoteWithStakeDelegations {

From f55e340542f0f7fc3c45e7ba1da2a626bb9b440e Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 3 Sep 2022 17:04:28 +0900
Subject: [PATCH 1859/3199] save

---
 runtime/src/bank.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 031485f0ab1622..44815714938f23 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1387,7 +1387,7 @@ pub struct Bank {
 
     pub incremental_snapshot_persistence: Option<BankIncrementalSnapshotPersistence>,
 
-    scheduler: ArcSwap<Scheduler>,
+    scheduler: RwLock<Scheduler>,
 }
 
 struct VoteWithStakeDelegations {

From dbf2620968ea49bd20b1dfb452cb641855f0dfdf Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 3 Sep 2022 17:05:49 +0900
Subject: [PATCH 1860/3199] save

---
 runtime/src/bank.rs | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 44815714938f23..0b40f724eb1554 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1578,7 +1578,7 @@ impl Bank {
             accounts_data_size_delta_on_chain: AtomicI64::new(0),
             accounts_data_size_delta_off_chain: AtomicI64::new(0),
             fee_structure: FeeStructure::default(),
-            scheduler: Scheduler::default(),
+            scheduler: Default::default(),
         };
 
         let accounts_data_size_initial = bank.get_total_accounts_stats().unwrap().data_len as u64;
@@ -1818,7 +1818,7 @@ impl Bank {
             measure!(parent.feature_set.clone(), "feature_set_creation");
 
         let accounts_data_size_initial = parent.load_accounts_data_size();
-        let scheduler = Scheduler::default();
+        let scheduler = Default::default();
 
         let mut new = Bank {
             incremental_snapshot_persistence: None,
@@ -2244,7 +2244,7 @@ impl Bank {
             accounts_data_size_delta_on_chain: AtomicI64::new(0),
             accounts_data_size_delta_off_chain: AtomicI64::new(0),
             fee_structure: FeeStructure::default(),
-            scheduler: Scheduler::default(),
+            scheduler: Default::default(),
         };
         bank.finish_init(
             genesis_config,

From 84f21b4e3e0aba3d85e244954ba82c6304daefa7 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 3 Sep 2022 17:06:11 +0900
Subject: [PATCH 1861/3199] save

---
 runtime/src/bank.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 0b40f724eb1554..3050bf51c6f78f 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -7869,7 +7869,7 @@ impl Bank {
     }
 
     fn wait_for_scheduler(&self) {
-        self.scheduler.scheduler_thread_handle.join();
+        self.scheduler.write().unwrap().scheduler_thread_handle.join();
     }
 }
 

From 8ee645e8c28fc22ca064ff11cf7e3b14785ce0d5 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 3 Sep 2022 17:06:38 +0900
Subject: [PATCH 1862/3199] save

---
 runtime/src/bank.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 3050bf51c6f78f..fea5c8831d12a1 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1198,7 +1198,7 @@ impl AbiExample for BuiltinPrograms {
 
 #[derive(Debug)]
 struct Scheduler {
-    scheduler_thread_handle: std::thread::JoinHandle<()>,
+    scheduler_thread_handle: Option<std::thread::JoinHandle<()>>,
 }
 
 impl Default for Scheduler {

From 93cd2f2f03e87ff467cb87d6ededf3f0fff39d28 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 3 Sep 2022 17:07:02 +0900
Subject: [PATCH 1863/3199] save

---
 runtime/src/bank.rs | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index fea5c8831d12a1..46ea2112511cc5 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -7869,7 +7869,8 @@ impl Bank {
     }
 
     fn wait_for_scheduler(&self) {
-        self.scheduler.write().unwrap().scheduler_thread_handle.join();
+        let h = self.scheduler.write().unwrap().scheduler_thread_handle.take();
+        h.join();
     }
 }
 

From d7d14ba70d17aca5eacda5cb614b1b5adb7e1c8d Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 3 Sep 2022 17:07:32 +0900
Subject: [PATCH 1864/3199] save

---
 runtime/src/bank.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 46ea2112511cc5..a1afec29f88b15 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1207,7 +1207,7 @@ impl Default for Scheduler {
         }).unwrap();
 
         Self {
-            scheduler_thread_handle
+            scheduler_thread_handle: Some(scheduler_thread_handle),
         }
     }
 }

From c24532350b6c4e3519624cb2a6789e952e17f4ce Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 3 Sep 2022 17:08:02 +0900
Subject: [PATCH 1865/3199] save

---
 runtime/src/bank.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index a1afec29f88b15..eee3c8a5508ec9 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -7869,7 +7869,7 @@ impl Bank {
     }
 
     fn wait_for_scheduler(&self) {
-        let h = self.scheduler.write().unwrap().scheduler_thread_handle.take();
+        let h = self.scheduler.write().unwrap().scheduler_thread_handle.take().unwrap();
         h.join();
     }
 }

From 578e83381f609f5715930ed975ef3310cb77f09a Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 3 Sep 2022 17:08:24 +0900
Subject: [PATCH 1866/3199] save

---
 runtime/src/bank.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index eee3c8a5508ec9..85b1750340d44c 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -7868,7 +7868,7 @@ impl Bank {
         total_accounts_stats
     }
 
-    fn wait_for_scheduler(&self) {
+    pub fn wait_for_scheduler(&self) {
         let h = self.scheduler.write().unwrap().scheduler_thread_handle.take().unwrap();
         h.join();
     }

From 23fab6e8425df978bca84b7f0ed77932681dde9e Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 3 Sep 2022 17:09:35 +0900
Subject: [PATCH 1867/3199] save

---
 core/src/replay_stage.rs | 23 ++++++++++++++++++++++-
 1 file changed, 22 insertions(+), 1 deletion(-)

diff --git a/core/src/replay_stage.rs b/core/src/replay_stage.rs
index dfe5674c17f2b8..73491161fff1c9 100644
--- a/core/src/replay_stage.rs
+++ b/core/src/replay_stage.rs
@@ -2472,7 +2472,28 @@ impl ReplayStage {
 
             assert_eq!(bank_slot, bank.slot());
             if bank.is_complete() {
-                bank.wait_for_scheduler();
+                let r = bank.wait_for_scheduler();
+                if let Err(err) = r {
+                        // Error means the slot needs to be marked as dead
+                        Self::mark_dead_slot(
+                            blockstore,
+                            bank,
+                            bank_forks.read().unwrap().root(),
+                            err,
+                            rpc_subscriptions,
+                            duplicate_slots_tracker,
+                            gossip_duplicate_confirmed_slots,
+                            epoch_slots_frozen_slots,
+                            progress,
+                            heaviest_subtree_fork_choice,
+                            duplicate_slots_to_repair,
+                            ancestor_hashes_replay_update_sender,
+                        );
+                        // If the bank was corrupted, don't try to run the below logic to check if the
+                        // bank is completed
+                        continue;
+                }
+
                 let mut bank_complete_time = Measure::start("bank_complete_time");
                 let bank_progress = progress
                     .get_mut(&bank.slot())

From 8746e126527d353e24849cbb5d9da05df2a0e822 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 3 Sep 2022 17:10:44 +0900
Subject: [PATCH 1868/3199] save

---
 runtime/src/bank.rs | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 85b1750340d44c..da7d011a7827f5 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -7868,9 +7868,10 @@ impl Bank {
         total_accounts_stats
     }
 
-    pub fn wait_for_scheduler(&self) {
+    pub fn wait_for_scheduler(&self) -> Result<(), ()> {
         let h = self.scheduler.write().unwrap().scheduler_thread_handle.take().unwrap();
         h.join();
+        Ok()
     }
 }
 

From 9eb3b9a2e949979c3c7294dde90e7be328a4a7e3 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 3 Sep 2022 17:11:20 +0900
Subject: [PATCH 1869/3199] save

---
 runtime/src/bank.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index da7d011a7827f5..865188bbb3a269 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -7868,10 +7868,10 @@ impl Bank {
         total_accounts_stats
     }
 
-    pub fn wait_for_scheduler(&self) -> Result<(), ()> {
+    pub fn wait_for_scheduler(&self) -> Result<()> {
         let h = self.scheduler.write().unwrap().scheduler_thread_handle.take().unwrap();
         h.join();
-        Ok()
+        Ok(())
     }
 }
 

From 97f118df6cfeb8ad9b7c4bbfd2a2ad6dfb38c576 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 3 Sep 2022 17:11:53 +0900
Subject: [PATCH 1870/3199] save

---
 core/src/replay_stage.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/core/src/replay_stage.rs b/core/src/replay_stage.rs
index 73491161fff1c9..80a85943a9f6e1 100644
--- a/core/src/replay_stage.rs
+++ b/core/src/replay_stage.rs
@@ -2479,7 +2479,7 @@ impl ReplayStage {
                             blockstore,
                             bank,
                             bank_forks.read().unwrap().root(),
-                            err,
+                            err.into(),
                             rpc_subscriptions,
                             duplicate_slots_tracker,
                             gossip_duplicate_confirmed_slots,

From 7da06016b383c63babc58da72c733d9a9c232912 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 3 Sep 2022 17:13:54 +0900
Subject: [PATCH 1871/3199] save

---
 core/src/replay_stage.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/core/src/replay_stage.rs b/core/src/replay_stage.rs
index 80a85943a9f6e1..10ceabab2f9661 100644
--- a/core/src/replay_stage.rs
+++ b/core/src/replay_stage.rs
@@ -2479,7 +2479,7 @@ impl ReplayStage {
                             blockstore,
                             bank,
                             bank_forks.read().unwrap().root(),
-                            err.into(),
+                            BlockstoreError::InvalidTransaction(err.into()),
                             rpc_subscriptions,
                             duplicate_slots_tracker,
                             gossip_duplicate_confirmed_slots,

From e5486ab4cf2e333341ecbef602b44a8f7c282302 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 3 Sep 2022 17:14:26 +0900
Subject: [PATCH 1872/3199] save

---
 core/src/replay_stage.rs | 1 +
 1 file changed, 1 insertion(+)

diff --git a/core/src/replay_stage.rs b/core/src/replay_stage.rs
index 10ceabab2f9661..5aaff5760a383c 100644
--- a/core/src/replay_stage.rs
+++ b/core/src/replay_stage.rs
@@ -2474,6 +2474,7 @@ impl ReplayStage {
             if bank.is_complete() {
                 let r = bank.wait_for_scheduler();
                 if let Err(err) = r {
+                    use solana_ledger::blockstore_db::BlockstoreError;
                         // Error means the slot needs to be marked as dead
                         Self::mark_dead_slot(
                             blockstore,

From 7b7016693cff48976802f41597c4013028bf6c0c Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 3 Sep 2022 17:14:53 +0900
Subject: [PATCH 1873/3199] save

---
 core/src/replay_stage.rs | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/core/src/replay_stage.rs b/core/src/replay_stage.rs
index 5aaff5760a383c..e5b674b8edb349 100644
--- a/core/src/replay_stage.rs
+++ b/core/src/replay_stage.rs
@@ -2474,13 +2474,12 @@ impl ReplayStage {
             if bank.is_complete() {
                 let r = bank.wait_for_scheduler();
                 if let Err(err) = r {
-                    use solana_ledger::blockstore_db::BlockstoreError;
                         // Error means the slot needs to be marked as dead
                         Self::mark_dead_slot(
                             blockstore,
                             bank,
                             bank_forks.read().unwrap().root(),
-                            BlockstoreError::InvalidTransaction(err.into()),
+                            BlockstoreProcessorError::InvalidTransaction(err.into()),
                             rpc_subscriptions,
                             duplicate_slots_tracker,
                             gossip_duplicate_confirmed_slots,

From 0272e55a1b2dbf7807f00f798dd46d44e94234d9 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 3 Sep 2022 17:15:04 +0900
Subject: [PATCH 1874/3199] save

---
 core/src/replay_stage.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/core/src/replay_stage.rs b/core/src/replay_stage.rs
index e5b674b8edb349..0f527794779f71 100644
--- a/core/src/replay_stage.rs
+++ b/core/src/replay_stage.rs
@@ -2479,7 +2479,7 @@ impl ReplayStage {
                             blockstore,
                             bank,
                             bank_forks.read().unwrap().root(),
-                            BlockstoreProcessorError::InvalidTransaction(err.into()),
+                            &BlockstoreProcessorError::InvalidTransaction(err.into()),
                             rpc_subscriptions,
                             duplicate_slots_tracker,
                             gossip_duplicate_confirmed_slots,

From d7c759f14e70c746bfe4cf86f81bf003943cbf5b Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 3 Sep 2022 17:15:37 +0900
Subject: [PATCH 1875/3199] save

---
 core/src/replay_stage.rs | 36 ++++++++++++++++++------------------
 runtime/src/bank.rs      |  3 ++-
 2 files changed, 20 insertions(+), 19 deletions(-)

diff --git a/core/src/replay_stage.rs b/core/src/replay_stage.rs
index 0f527794779f71..09d309de6cdafc 100644
--- a/core/src/replay_stage.rs
+++ b/core/src/replay_stage.rs
@@ -2474,24 +2474,24 @@ impl ReplayStage {
             if bank.is_complete() {
                 let r = bank.wait_for_scheduler();
                 if let Err(err) = r {
-                        // Error means the slot needs to be marked as dead
-                        Self::mark_dead_slot(
-                            blockstore,
-                            bank,
-                            bank_forks.read().unwrap().root(),
-                            &BlockstoreProcessorError::InvalidTransaction(err.into()),
-                            rpc_subscriptions,
-                            duplicate_slots_tracker,
-                            gossip_duplicate_confirmed_slots,
-                            epoch_slots_frozen_slots,
-                            progress,
-                            heaviest_subtree_fork_choice,
-                            duplicate_slots_to_repair,
-                            ancestor_hashes_replay_update_sender,
-                        );
-                        // If the bank was corrupted, don't try to run the below logic to check if the
-                        // bank is completed
-                        continue;
+                    // Error means the slot needs to be marked as dead
+                    Self::mark_dead_slot(
+                        blockstore,
+                        bank,
+                        bank_forks.read().unwrap().root(),
+                        &BlockstoreProcessorError::InvalidTransaction(err.into()),
+                        rpc_subscriptions,
+                        duplicate_slots_tracker,
+                        gossip_duplicate_confirmed_slots,
+                        epoch_slots_frozen_slots,
+                        progress,
+                        heaviest_subtree_fork_choice,
+                        duplicate_slots_to_repair,
+                        ancestor_hashes_replay_update_sender,
+                    );
+                    // If the bank was corrupted, don't try to run the below logic to check if the
+                    // bank is completed
+                    continue;
                 }
 
                 let mut bank_complete_time = Measure::start("bank_complete_time");
diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 865188bbb3a269..0cbb6ef841c54a 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -7870,7 +7870,8 @@ impl Bank {
 
     pub fn wait_for_scheduler(&self) -> Result<()> {
         let h = self.scheduler.write().unwrap().scheduler_thread_handle.take().unwrap();
-        h.join();
+        h.join()?;
+
         Ok(())
     }
 }

From 460fd2a5afce38fde40366b209be567b87517314 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 3 Sep 2022 17:16:03 +0900
Subject: [PATCH 1876/3199] save

---
 runtime/src/bank.rs | 1 +
 1 file changed, 1 insertion(+)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 0cbb6ef841c54a..245a8c44bbb9c8 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1204,6 +1204,7 @@ struct Scheduler {
 impl Default for Scheduler {
     fn default() -> Self {
         let scheduler_thread_handle = std::thread::Builder::new().name("sol-scheduler".to_string()).spawn(move || {
+            Ok(())
         }).unwrap();
 
         Self {

From 41aaadb78f82ac1169fdf92dc103b1866a78d0b5 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 3 Sep 2022 17:16:26 +0900
Subject: [PATCH 1877/3199] save

---
 runtime/src/bank.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 245a8c44bbb9c8..97d855c3dbc68e 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1198,7 +1198,7 @@ impl AbiExample for BuiltinPrograms {
 
 #[derive(Debug)]
 struct Scheduler {
-    scheduler_thread_handle: Option<std::thread::JoinHandle<()>>,
+    scheduler_thread_handle: Option<std::thread::JoinHandle<Result<()>>>,
 }
 
 impl Default for Scheduler {

From b16cdfd5be75dbda8202c5d5c6f7fc7be5e89573 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 3 Sep 2022 17:17:12 +0900
Subject: [PATCH 1878/3199] save

---
 runtime/src/bank.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 97d855c3dbc68e..df63d8d32e3057 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -7871,7 +7871,7 @@ impl Bank {
 
     pub fn wait_for_scheduler(&self) -> Result<()> {
         let h = self.scheduler.write().unwrap().scheduler_thread_handle.take().unwrap();
-        h.join()?;
+        let a: usize = h.join();
 
         Ok(())
     }

From 2efa2d3533268ce1088d6bbe05692fcb182dd9f6 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 3 Sep 2022 17:17:27 +0900
Subject: [PATCH 1879/3199] save

---
 runtime/src/bank.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index df63d8d32e3057..8f0e0d6a2716f3 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -7871,7 +7871,7 @@ impl Bank {
 
     pub fn wait_for_scheduler(&self) -> Result<()> {
         let h = self.scheduler.write().unwrap().scheduler_thread_handle.take().unwrap();
-        let a: usize = h.join();
+        let a: usize = h.join().unwrap();
 
         Ok(())
     }

From 12b87fef263c0d9a0af8251e3444d6413ccd0c44 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 3 Sep 2022 17:17:41 +0900
Subject: [PATCH 1880/3199] save

---
 runtime/src/bank.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 8f0e0d6a2716f3..4a4afc052c201b 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -7871,7 +7871,7 @@ impl Bank {
 
     pub fn wait_for_scheduler(&self) -> Result<()> {
         let h = self.scheduler.write().unwrap().scheduler_thread_handle.take().unwrap();
-        let a: usize = h.join().unwrap();
+        h.join().unwrap()?;
 
         Ok(())
     }

From d0e57dd3c164a81d32950695b9ed2e4dbf7d13ab Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 3 Sep 2022 20:02:36 +0900
Subject: [PATCH 1881/3199] save

---
 runtime/src/bank.rs  | 1 +
 scheduler/Cargo.toml | 4 ++--
 scheduler/src/lib.rs | 2 ++
 3 files changed, 5 insertions(+), 2 deletions(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 4a4afc052c201b..bc59dd72280344 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1204,6 +1204,7 @@ struct Scheduler {
 impl Default for Scheduler {
     fn default() -> Self {
         let scheduler_thread_handle = std::thread::Builder::new().name("sol-scheduler".to_string()).spawn(move || {
+            let mut runnable_queue = solana_scheduler::TaskQueue::default();
             Ok(())
         }).unwrap();
 
diff --git a/scheduler/Cargo.toml b/scheduler/Cargo.toml
index 06f9fb2a45354d..52f2d252f6285f 100644
--- a/scheduler/Cargo.toml
+++ b/scheduler/Cargo.toml
@@ -16,8 +16,8 @@ crossbeam-skiplist = { path = "../../../crossbeam/crossbeam-skiplist" }
 dashmap = { version = "4.0.2" }
 log = "0.4.17"
 solana-entry = { path = "../entry", version = "=1.12.0" }
-solana-runtime = { path = "../runtime", version = "=1.12.0" }
-solana-program-runtime = { path = "../program-runtime", version = "=1.12.0" }
+#solana-runtime = { path = "../runtime", version = "=1.12.0" }
+#solana-program-runtime = { path = "../program-runtime", version = "=1.12.0" }
 solana-sdk = { path = "../sdk", version = "=1.12.0" }
 solana-logger = { path = "../logger", version = "=1.12.0" }
 solana-version = { path = "../version", version = "=1.12.0" }
diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 24ee4784825453..7d2e7b3fa27a24 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -792,6 +792,7 @@ fn attempt_lock_for_execution<'a, AST: AtScheduleThread>(
 
 type PreprocessedTransaction = (SanitizedTransaction, Vec<LockAttempt>);
 
+/*
 pub fn get_transaction_priority_details(tx: &SanitizedTransaction) -> u64 {
     use solana_program_runtime::compute_budget::ComputeBudget;
     let mut compute_budget = ComputeBudget::default();
@@ -803,6 +804,7 @@ pub fn get_transaction_priority_details(tx: &SanitizedTransaction) -> u64 {
         .map(|d| d.get_priority())
         .unwrap_or_default()
 }
+*/
 
 pub struct ScheduleStage {}
 

From c852e16d81b0a82523d8c4eef02a9f9b4fb90111 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 3 Sep 2022 20:02:52 +0900
Subject: [PATCH 1882/3199] save

---
 Cargo.lock          | 2 --
 runtime/src/bank.rs | 2 +-
 2 files changed, 1 insertion(+), 3 deletions(-)

diff --git a/Cargo.lock b/Cargo.lock
index 2d95ffafca1321..f727b03520130a 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -6200,8 +6200,6 @@ dependencies = [
  "solana-logger 1.12.0",
  "solana-measure",
  "solana-metrics",
- "solana-program-runtime",
- "solana-runtime",
  "solana-sdk 1.12.0",
  "solana-version",
  "triomphe",
diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index bc59dd72280344..f60103f175e29c 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1204,7 +1204,7 @@ struct Scheduler {
 impl Default for Scheduler {
     fn default() -> Self {
         let scheduler_thread_handle = std::thread::Builder::new().name("sol-scheduler".to_string()).spawn(move || {
-            let mut runnable_queue = solana_scheduler::TaskQueue::default();
+            //let mut runnable_queue = solana_scheduler::TaskQueue::default();
             Ok(())
         }).unwrap();
 

From 3244f1191d641f618ce7229789669a76d1c6db81 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 3 Sep 2022 20:05:50 +0900
Subject: [PATCH 1883/3199] save

---
 Cargo.lock           | 2 +-
 runtime/Cargo.toml   | 1 +
 scheduler/Cargo.toml | 3 ---
 scheduler/src/lib.rs | 1 -
 4 files changed, 2 insertions(+), 5 deletions(-)

diff --git a/Cargo.lock b/Cargo.lock
index f727b03520130a..90b0cadb881347 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -6172,6 +6172,7 @@ dependencies = [
  "solana-metrics",
  "solana-program-runtime",
  "solana-rayon-threadlimit",
+ "solana-scheduler",
  "solana-sdk 1.12.0",
  "solana-stake-program",
  "solana-vote-program",
@@ -6196,7 +6197,6 @@ dependencies = [
  "log",
  "rand 0.7.3",
  "sha2 0.10.2",
- "solana-entry",
  "solana-logger 1.12.0",
  "solana-measure",
  "solana-metrics",
diff --git a/runtime/Cargo.toml b/runtime/Cargo.toml
index 251fc0ef1ccfc4..8b2ea6ce652572 100644
--- a/runtime/Cargo.toml
+++ b/runtime/Cargo.toml
@@ -50,6 +50,7 @@ solana-measure = { path = "../measure", version = "=1.12.0" }
 solana-metrics = { path = "../metrics", version = "=1.12.0" }
 solana-program-runtime = { path = "../program-runtime", version = "=1.12.0" }
 solana-rayon-threadlimit = { path = "../rayon-threadlimit", version = "=1.12.0" }
+solana-scheduler = { path = "../scheduler", version = "=1.12.0" }
 solana-sdk = { path = "../sdk", version = "=1.12.0" }
 solana-stake-program = { path = "../programs/stake", version = "=1.12.0" }
 solana-vote-program = { path = "../programs/vote", version = "=1.12.0" }
diff --git a/scheduler/Cargo.toml b/scheduler/Cargo.toml
index 52f2d252f6285f..4b1e00b0c553ed 100644
--- a/scheduler/Cargo.toml
+++ b/scheduler/Cargo.toml
@@ -15,9 +15,6 @@ crossbeam-channel = "0.5.5"
 crossbeam-skiplist = { path = "../../../crossbeam/crossbeam-skiplist" }
 dashmap = { version = "4.0.2" }
 log = "0.4.17"
-solana-entry = { path = "../entry", version = "=1.12.0" }
-#solana-runtime = { path = "../runtime", version = "=1.12.0" }
-#solana-program-runtime = { path = "../program-runtime", version = "=1.12.0" }
 solana-sdk = { path = "../sdk", version = "=1.12.0" }
 solana-logger = { path = "../logger", version = "=1.12.0" }
 solana-version = { path = "../version", version = "=1.12.0" }
diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 7d2e7b3fa27a24..7b0ac5ded4101b 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -5,7 +5,6 @@ use {
     log::*,
     rand::Rng,
     sha2::{Digest, Sha256},
-    solana_entry::entry::Entry,
     solana_measure::measure::Measure,
     solana_metrics::datapoint_info,
     solana_sdk::{

From d659759a2c0edea271ea99c095b70ae5a9679e5e Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 3 Sep 2022 20:06:11 +0900
Subject: [PATCH 1884/3199] save

---
 runtime/src/bank.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index f60103f175e29c..bc59dd72280344 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1204,7 +1204,7 @@ struct Scheduler {
 impl Default for Scheduler {
     fn default() -> Self {
         let scheduler_thread_handle = std::thread::Builder::new().name("sol-scheduler".to_string()).spawn(move || {
-            //let mut runnable_queue = solana_scheduler::TaskQueue::default();
+            let mut runnable_queue = solana_scheduler::TaskQueue::default();
             Ok(())
         }).unwrap();
 

From 3f6d969622b78664f9cc2df9632977316ffdb2cf Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 3 Sep 2022 20:07:08 +0900
Subject: [PATCH 1885/3199] save

---
 runtime/src/bank.rs | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index bc59dd72280344..5802813f052041 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1203,8 +1203,11 @@ struct Scheduler {
 
 impl Default for Scheduler {
     fn default() -> Self {
+        let mut runnable_queue = solana_scheduler::TaskQueue::default();
+        let mut address_book = solana_scheduler::AddressBook::default();
+        let preloader = address_book.preloader();
+
         let scheduler_thread_handle = std::thread::Builder::new().name("sol-scheduler".to_string()).spawn(move || {
-            let mut runnable_queue = solana_scheduler::TaskQueue::default();
             Ok(())
         }).unwrap();
 

From caeabaac09eff17e9c18fa5316b0aef46dd51f07 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 3 Sep 2022 20:07:43 +0900
Subject: [PATCH 1886/3199] save

---
 runtime/src/bank.rs | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 5802813f052041..6b22e81e00f3f8 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1199,6 +1199,7 @@ impl AbiExample for BuiltinPrograms {
 #[derive(Debug)]
 struct Scheduler {
     scheduler_thread_handle: Option<std::thread::JoinHandle<Result<()>>>,
+    preloader: usize,
 }
 
 impl Default for Scheduler {
@@ -1213,6 +1214,7 @@ impl Default for Scheduler {
 
         Self {
             scheduler_thread_handle: Some(scheduler_thread_handle),
+            preloader,
         }
     }
 }

From 6141c2a5aac1a2222ebfdf0f77bac905d7aa5e25 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 3 Sep 2022 20:08:03 +0900
Subject: [PATCH 1887/3199] save

---
 runtime/src/bank.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 6b22e81e00f3f8..90cc42c285a66b 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1206,7 +1206,7 @@ impl Default for Scheduler {
     fn default() -> Self {
         let mut runnable_queue = solana_scheduler::TaskQueue::default();
         let mut address_book = solana_scheduler::AddressBook::default();
-        let preloader = address_book.preloader();
+        let preloader = Arc::new(address_book.preloader());
 
         let scheduler_thread_handle = std::thread::Builder::new().name("sol-scheduler".to_string()).spawn(move || {
             Ok(())

From 1dcef473ea4f09207548a97e85c980c326e16c55 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 3 Sep 2022 20:08:30 +0900
Subject: [PATCH 1888/3199] save

---
 runtime/src/bank.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 90cc42c285a66b..3b292e9d9446a3 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1199,7 +1199,7 @@ impl AbiExample for BuiltinPrograms {
 #[derive(Debug)]
 struct Scheduler {
     scheduler_thread_handle: Option<std::thread::JoinHandle<Result<()>>>,
-    preloader: usize,
+    preloader: Arc<solana_scheduler::Preloader>,
 }
 
 impl Default for Scheduler {

From c9eab50c246dae6f7e3b44f6f4d28e93ef1fea5f Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 3 Sep 2022 20:08:53 +0900
Subject: [PATCH 1889/3199] save

---
 scheduler/src/lib.rs | 1 +
 1 file changed, 1 insertion(+)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 7b0ac5ded4101b..de1ad039e6aa01 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -446,6 +446,7 @@ impl AddressBook {
     }
 }
 
+#[derive(Debug)]
 pub struct Preloader {
     book: AddressMap,
 }

From b63e0ee32149e66ac5a9cbf0375e1acea5f61a92 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 3 Sep 2022 20:09:38 +0900
Subject: [PATCH 1890/3199] save

---
 runtime/src/bank.rs | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 3b292e9d9446a3..aa9305fe910705 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1209,6 +1209,17 @@ impl Default for Scheduler {
         let preloader = Arc::new(address_book.preloader());
 
         let scheduler_thread_handle = std::thread::Builder::new().name("sol-scheduler".to_string()).spawn(move || {
+            let mut runnable_queue = solana_scheduler::TaskQueue::default();
+
+            solana_scheduler::ScheduleStage::run(
+                num_execution_threads * lane_channel_factor,
+                &mut runnable_queue,
+                &mut address_book,
+                &packet_batch_receiver.clone(),
+                &completed_transaction_receiver,
+                &transaction_batch_senders[0],
+                None,//&completed_transaction_receiver
+            );
             Ok(())
         }).unwrap();
 

From 0930b40c55c97bfa9f64beb3cc5e9bddd52bb9f1 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 3 Sep 2022 20:10:50 +0900
Subject: [PATCH 1891/3199] save

---
 runtime/src/bank.rs | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index aa9305fe910705..3bac2edc6fa7bf 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1207,17 +1207,20 @@ impl Default for Scheduler {
         let mut runnable_queue = solana_scheduler::TaskQueue::default();
         let mut address_book = solana_scheduler::AddressBook::default();
         let preloader = Arc::new(address_book.preloader());
+        let (packet_batch_sender, packet_batch_receiver) = crossbeam_channel::unbounded();
+        let (completed_transaction_sender, completed_transaction_receiver) = crossbeam_channel::unbounded();
+        let (transaction_batch_sender, transaction_batch_receiver) = crossbeam_channel::unbounded();
 
         let scheduler_thread_handle = std::thread::Builder::new().name("sol-scheduler".to_string()).spawn(move || {
             let mut runnable_queue = solana_scheduler::TaskQueue::default();
 
             solana_scheduler::ScheduleStage::run(
-                num_execution_threads * lane_channel_factor,
+                100,
                 &mut runnable_queue,
                 &mut address_book,
                 &packet_batch_receiver.clone(),
                 &completed_transaction_receiver,
-                &transaction_batch_senders[0],
+                &transaction_batch_sender,
                 None,//&completed_transaction_receiver
             );
             Ok(())

From b7577776035372d34710202fd3419fa90b302b3d Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 3 Sep 2022 20:12:32 +0900
Subject: [PATCH 1892/3199] save

---
 runtime/src/bank.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 3bac2edc6fa7bf..3bb5ba4526009c 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1211,7 +1211,7 @@ impl Default for Scheduler {
         let (completed_transaction_sender, completed_transaction_receiver) = crossbeam_channel::unbounded();
         let (transaction_batch_sender, transaction_batch_receiver) = crossbeam_channel::unbounded();
 
-        let scheduler_thread_handle = std::thread::Builder::new().name("sol-scheduler".to_string()).spawn(move || {
+        let scheduler_thread_handle = std::thread::Builder::new().name("solScheduler".to_string()).spawn(move || {
             let mut runnable_queue = solana_scheduler::TaskQueue::default();
 
             solana_scheduler::ScheduleStage::run(

From 5c98f5d2fceb2e11ab32a1fc7f15986487db4ed0 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 3 Sep 2022 22:57:00 +0900
Subject: [PATCH 1893/3199] save

---
 runtime/src/bank.rs | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 3bb5ba4526009c..d8b6b883516d1f 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1213,6 +1213,8 @@ impl Default for Scheduler {
 
         let scheduler_thread_handle = std::thread::Builder::new().name("solScheduler".to_string()).spawn(move || {
             let mut runnable_queue = solana_scheduler::TaskQueue::default();
+            let e = std::thread::Builder::new().name(format!("sol-exec-{}", thread_index)).spawn(move || {
+            }).unwrap();
 
             solana_scheduler::ScheduleStage::run(
                 100,

From cd2e0e1909967046bf4dafa8a241f2368386f729 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 3 Sep 2022 22:57:16 +0900
Subject: [PATCH 1894/3199] save

---
 runtime/src/bank.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index d8b6b883516d1f..b06c38b39d46bb 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1213,7 +1213,7 @@ impl Default for Scheduler {
 
         let scheduler_thread_handle = std::thread::Builder::new().name("solScheduler".to_string()).spawn(move || {
             let mut runnable_queue = solana_scheduler::TaskQueue::default();
-            let e = std::thread::Builder::new().name(format!("sol-exec-{}", thread_index)).spawn(move || {
+            let e = std::thread::Builder::new().name(format!("sol-exec-{}", 0)).spawn(move || {
             }).unwrap();
 
             solana_scheduler::ScheduleStage::run(

From 68b915f1c53aa9a94575bceeff928833375ea81e Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 3 Sep 2022 22:59:24 +0900
Subject: [PATCH 1895/3199] save

---
 runtime/src/bank.rs | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index b06c38b39d46bb..54b17f01e2122f 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1198,7 +1198,8 @@ impl AbiExample for BuiltinPrograms {
 
 #[derive(Debug)]
 struct Scheduler {
-    scheduler_thread_handle: Option<std::thread::JoinHandle<Result<()>>>,
+    cheduler_thread_handle: Option<std::thread::JoinHandle<Result<()>>>,
+    executing_thread_handle: Option<std::thread::JoinHandle<()>>,
     preloader: Arc<solana_scheduler::Preloader>,
 }
 
@@ -1210,11 +1211,11 @@ impl Default for Scheduler {
         let (packet_batch_sender, packet_batch_receiver) = crossbeam_channel::unbounded();
         let (completed_transaction_sender, completed_transaction_receiver) = crossbeam_channel::unbounded();
         let (transaction_batch_sender, transaction_batch_receiver) = crossbeam_channel::unbounded();
+        let executing_thread_handle = std::thread::Builder::new().name(format!("sol-exec-{}", 0)).spawn(move || {
+        }).unwrap();
 
         let scheduler_thread_handle = std::thread::Builder::new().name("solScheduler".to_string()).spawn(move || {
             let mut runnable_queue = solana_scheduler::TaskQueue::default();
-            let e = std::thread::Builder::new().name(format!("sol-exec-{}", 0)).spawn(move || {
-            }).unwrap();
 
             solana_scheduler::ScheduleStage::run(
                 100,
@@ -1230,6 +1231,7 @@ impl Default for Scheduler {
 
         Self {
             scheduler_thread_handle: Some(scheduler_thread_handle),
+            executing_thread_handle,
             preloader,
         }
     }

From 90a7d40bc58d15859e3188ac41ab743091395cfc Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 3 Sep 2022 22:59:51 +0900
Subject: [PATCH 1896/3199] save

---
 runtime/src/bank.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 54b17f01e2122f..e02ee0d8f44b4a 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1231,7 +1231,7 @@ impl Default for Scheduler {
 
         Self {
             scheduler_thread_handle: Some(scheduler_thread_handle),
-            executing_thread_handle,
+            executing_thread_handle: Some(executing_thread_handle),
             preloader,
         }
     }

From 5541acb61118aac2ecf79b0bc9e9f5bf550987d3 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 3 Sep 2022 23:00:13 +0900
Subject: [PATCH 1897/3199] save

---
 runtime/src/bank.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index e02ee0d8f44b4a..e67ff49a3dee39 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1198,7 +1198,7 @@ impl AbiExample for BuiltinPrograms {
 
 #[derive(Debug)]
 struct Scheduler {
-    cheduler_thread_handle: Option<std::thread::JoinHandle<Result<()>>>,
+    scheduler_thread_handle: Option<std::thread::JoinHandle<Result<()>>>,
     executing_thread_handle: Option<std::thread::JoinHandle<()>>,
     preloader: Arc<solana_scheduler::Preloader>,
 }

From 554ceb2cc11a66ff9b677ab0a3b69a5ca7af4ae3 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 3 Sep 2022 23:01:52 +0900
Subject: [PATCH 1898/3199] save

---
 runtime/src/bank.rs | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index e67ff49a3dee39..702a45826f1377 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1200,6 +1200,7 @@ impl AbiExample for BuiltinPrograms {
 struct Scheduler {
     scheduler_thread_handle: Option<std::thread::JoinHandle<Result<()>>>,
     executing_thread_handle: Option<std::thread::JoinHandle<()>>,
+    transaction_sender: Option<usize>,
     preloader: Arc<solana_scheduler::Preloader>,
 }
 
@@ -1232,6 +1233,7 @@ impl Default for Scheduler {
         Self {
             scheduler_thread_handle: Some(scheduler_thread_handle),
             executing_thread_handle: Some(executing_thread_handle),
+            transaction_sender: Some(transaction_sender),
             preloader,
         }
     }

From b6b70f5d550b78aa9a3330194638aa648cad1c0f Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 3 Sep 2022 23:02:11 +0900
Subject: [PATCH 1899/3199] save

---
 runtime/src/bank.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 702a45826f1377..85116488a0653b 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1209,7 +1209,7 @@ impl Default for Scheduler {
         let mut runnable_queue = solana_scheduler::TaskQueue::default();
         let mut address_book = solana_scheduler::AddressBook::default();
         let preloader = Arc::new(address_book.preloader());
-        let (packet_batch_sender, packet_batch_receiver) = crossbeam_channel::unbounded();
+        let (transaction_sender, transaction_receiver) = crossbeam_channel::unbounded();
         let (completed_transaction_sender, completed_transaction_receiver) = crossbeam_channel::unbounded();
         let (transaction_batch_sender, transaction_batch_receiver) = crossbeam_channel::unbounded();
         let executing_thread_handle = std::thread::Builder::new().name(format!("sol-exec-{}", 0)).spawn(move || {

From ac99cd49de985d2ed8282089e9b8630298f8772e Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 3 Sep 2022 23:02:31 +0900
Subject: [PATCH 1900/3199] save

---
 runtime/src/bank.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 85116488a0653b..3d67bd7f1e0e2e 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1222,7 +1222,7 @@ impl Default for Scheduler {
                 100,
                 &mut runnable_queue,
                 &mut address_book,
-                &packet_batch_receiver.clone(),
+                &transaction_receiver.clone(),
                 &completed_transaction_receiver,
                 &transaction_batch_sender,
                 None,//&completed_transaction_receiver

From 38d15593295e36636116a96c217167a169c1fd15 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 3 Sep 2022 23:05:45 +0900
Subject: [PATCH 1901/3199] save

---
 runtime/src/bank.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 3d67bd7f1e0e2e..1e74d0f9c1d79f 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1200,7 +1200,7 @@ impl AbiExample for BuiltinPrograms {
 struct Scheduler {
     scheduler_thread_handle: Option<std::thread::JoinHandle<Result<()>>>,
     executing_thread_handle: Option<std::thread::JoinHandle<()>>,
-    transaction_sender: Option<usize>,
+    transaction_sender: Option<crossbeam_channel::Sender<solana_scheduler::TransactionPayload>>,
     preloader: Arc<solana_scheduler::Preloader>,
 }
 

From 55c1fc4b961e65f1d65bd5a8d31f5a23ffce5004 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 3 Sep 2022 23:07:35 +0900
Subject: [PATCH 1902/3199] save

---
 runtime/src/bank.rs  | 2 +-
 scheduler/src/lib.rs | 2 ++
 2 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 1e74d0f9c1d79f..98b4bf2f0ab11d 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1200,7 +1200,7 @@ impl AbiExample for BuiltinPrograms {
 struct Scheduler {
     scheduler_thread_handle: Option<std::thread::JoinHandle<Result<()>>>,
     executing_thread_handle: Option<std::thread::JoinHandle<()>>,
-    transaction_sender: Option<crossbeam_channel::Sender<solana_scheduler::TransactionPayload>>,
+    transaction_sender: Option<crossbeam_channel::Sender<solana_scheduler::SchedulablePayload>>,
     preloader: Arc<solana_scheduler::Preloader>,
 }
 
diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index de1ad039e6aa01..ed3e432e954087 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1467,6 +1467,8 @@ impl ScheduleStage {
     }
 }
 
+pub struct SchedulablePayload(TaskInQueue);
+
 struct ExecuteStage {
     //bank: Bank,
 }

From 56cdc7a9685bf900308b4682f74c4c97d92d5aa1 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 3 Sep 2022 23:08:13 +0900
Subject: [PATCH 1903/3199] save

---
 scheduler/src/lib.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index ed3e432e954087..c0cd6e3d334395 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1268,7 +1268,7 @@ impl ScheduleStage {
         max_executing_queue_count: usize,
         runnable_queue: &mut TaskQueue,
         address_book: &mut AddressBook,
-        from: &crossbeam_channel::Receiver<TaskInQueue>,
+        from: &crossbeam_channel::Receiver<SchedulablePayload>,
         from_exec: &crossbeam_channel::Receiver<Box<ExecutionEnvironment>>,
         to_execute_substage: &crossbeam_channel::Sender<Box<ExecutionEnvironment>>,
         maybe_to_next_stage: Option<&crossbeam_channel::Sender<Box<ExecutionEnvironment>>>, // assume nonblocking
@@ -1445,7 +1445,7 @@ impl ScheduleStage {
         max_executing_queue_count: usize,
         runnable_queue: &mut TaskQueue,
         address_book: &mut AddressBook,
-        from: &crossbeam_channel::Receiver<TaskInQueue>,
+        from: &crossbeam_channel::Receiver<SchedulablePayload>,
         from_exec: &crossbeam_channel::Receiver<Box<ExecutionEnvironment>>,
         to_execute_substage: &crossbeam_channel::Sender<Box<ExecutionEnvironment>>,
         maybe_to_next_stage: Option<&crossbeam_channel::Sender<Box<ExecutionEnvironment>>>, // assume nonblocking

From 67234f9d79fff8ac896f9e8e6928f4157d18e054 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 3 Sep 2022 23:08:27 +0900
Subject: [PATCH 1904/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index c0cd6e3d334395..1a21db4859aa7d 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1429,7 +1429,7 @@ impl ScheduleStage {
                         to_next_stage.send(processed_execution_environment).unwrap();
                     }
                     if !empty_from {
-                        let task = from.recv().unwrap();
+                        let task = from.recv().unwrap().0;
                         from_len = from_len.checked_sub(1).unwrap();
                         empty_from = from_len == 0;
                         Self::register_runnable_task(task, runnable_queue, &mut sequence_time);

From 0abc54909bd0af2046beed575cea18a5add0db68 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 3 Sep 2022 23:08:41 +0900
Subject: [PATCH 1905/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 1a21db4859aa7d..3ec11388a16051 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1360,7 +1360,7 @@ impl ScheduleStage {
                            continue;
                        }
                    }
-                   let task = maybe_from.unwrap();
+                   let task = maybe_from.unwrap().0;
                    Self::register_runnable_task(task, runnable_queue, &mut sequence_time);
                }
             }

From f9ac4cf01d082a033c6810244b7b5a4df0fd5a21 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 3 Sep 2022 23:09:56 +0900
Subject: [PATCH 1906/3199] save

---
 runtime/src/bank.rs  | 2 +-
 scheduler/src/lib.rs | 6 +++---
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 98b4bf2f0ab11d..a9062c71a89bf5 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1200,7 +1200,7 @@ impl AbiExample for BuiltinPrograms {
 struct Scheduler {
     scheduler_thread_handle: Option<std::thread::JoinHandle<Result<()>>>,
     executing_thread_handle: Option<std::thread::JoinHandle<()>>,
-    transaction_sender: Option<crossbeam_channel::Sender<solana_scheduler::SchedulablePayload>>,
+    transaction_sender: Option<crossbeam_channel::Sender<solana_scheduler::Schedulable>>,
     preloader: Arc<solana_scheduler::Preloader>,
 }
 
diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 3ec11388a16051..d25bd860f3b294 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1268,7 +1268,7 @@ impl ScheduleStage {
         max_executing_queue_count: usize,
         runnable_queue: &mut TaskQueue,
         address_book: &mut AddressBook,
-        from: &crossbeam_channel::Receiver<SchedulablePayload>,
+        from: &crossbeam_channel::Receiver<Schedulable>,
         from_exec: &crossbeam_channel::Receiver<Box<ExecutionEnvironment>>,
         to_execute_substage: &crossbeam_channel::Sender<Box<ExecutionEnvironment>>,
         maybe_to_next_stage: Option<&crossbeam_channel::Sender<Box<ExecutionEnvironment>>>, // assume nonblocking
@@ -1445,7 +1445,7 @@ impl ScheduleStage {
         max_executing_queue_count: usize,
         runnable_queue: &mut TaskQueue,
         address_book: &mut AddressBook,
-        from: &crossbeam_channel::Receiver<SchedulablePayload>,
+        from: &crossbeam_channel::Receiver<Schedulable>,
         from_exec: &crossbeam_channel::Receiver<Box<ExecutionEnvironment>>,
         to_execute_substage: &crossbeam_channel::Sender<Box<ExecutionEnvironment>>,
         maybe_to_next_stage: Option<&crossbeam_channel::Sender<Box<ExecutionEnvironment>>>, // assume nonblocking
@@ -1467,7 +1467,7 @@ impl ScheduleStage {
     }
 }
 
-pub struct SchedulablePayload(TaskInQueue);
+pub struct Schedulable(TaskInQueue);
 
 struct ExecuteStage {
     //bank: Bank,

From 22198f5d524528e96a23bf3bd943d3aa1cb029e9 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 3 Sep 2022 23:26:29 +0900
Subject: [PATCH 1907/3199] save

---
 scheduler/src/lib.rs | 15 +++++++++------
 1 file changed, 9 insertions(+), 6 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index d25bd860f3b294..43a03b8ca759d1 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1268,7 +1268,7 @@ impl ScheduleStage {
         max_executing_queue_count: usize,
         runnable_queue: &mut TaskQueue,
         address_book: &mut AddressBook,
-        from: &crossbeam_channel::Receiver<Schedulable>,
+        from: &crossbeam_channel::Receiver<SchedulablePayload>,
         from_exec: &crossbeam_channel::Receiver<Box<ExecutionEnvironment>>,
         to_execute_substage: &crossbeam_channel::Sender<Box<ExecutionEnvironment>>,
         maybe_to_next_stage: Option<&crossbeam_channel::Sender<Box<ExecutionEnvironment>>>, // assume nonblocking
@@ -1445,10 +1445,10 @@ impl ScheduleStage {
         max_executing_queue_count: usize,
         runnable_queue: &mut TaskQueue,
         address_book: &mut AddressBook,
-        from: &crossbeam_channel::Receiver<Schedulable>,
-        from_exec: &crossbeam_channel::Receiver<Box<ExecutionEnvironment>>,
-        to_execute_substage: &crossbeam_channel::Sender<Box<ExecutionEnvironment>>,
-        maybe_to_next_stage: Option<&crossbeam_channel::Sender<Box<ExecutionEnvironment>>>, // assume nonblocking
+        from: &crossbeam_channel::Receiver<SchedulablePayload>,
+        from_exec: &crossbeam_channel::Receiver<UnlockablePayload>,
+        to_execute_substage: &crossbeam_channel::Sender<ExecutablePayload>,
+        maybe_to_next_stage: Option<&crossbeam_channel::Sender<PersistablePlayload>, // assume nonblocking
     ) {
         #[derive(Clone, Copy, Debug)]
         struct AtTopOfScheduleThread;
@@ -1467,7 +1467,10 @@ impl ScheduleStage {
     }
 }
 
-pub struct Schedulable(TaskInQueue);
+pub struct SchedulablePayload(TaskInQueue);
+pub struct ExecutablePayload(Box<ExecutionEnvironment>)
+pub struct UnlockablePayload(Box<ExecutionEnvironment>);
+pub struct PersistablePlayload(Box<ExecutionEnvironment>);
 
 struct ExecuteStage {
     //bank: Bank,

From 6b2003ea48a000352d89b7a25149837237601a70 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 3 Sep 2022 23:26:55 +0900
Subject: [PATCH 1908/3199] save

---
 scheduler/src/lib.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 43a03b8ca759d1..1f21b1909aa200 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1448,7 +1448,7 @@ impl ScheduleStage {
         from: &crossbeam_channel::Receiver<SchedulablePayload>,
         from_exec: &crossbeam_channel::Receiver<UnlockablePayload>,
         to_execute_substage: &crossbeam_channel::Sender<ExecutablePayload>,
-        maybe_to_next_stage: Option<&crossbeam_channel::Sender<PersistablePlayload>, // assume nonblocking
+        maybe_to_next_stage: Option<&crossbeam_channel::Sender<PersistablePlayload>>, // assume nonblocking
     ) {
         #[derive(Clone, Copy, Debug)]
         struct AtTopOfScheduleThread;
@@ -1468,7 +1468,7 @@ impl ScheduleStage {
 }
 
 pub struct SchedulablePayload(TaskInQueue);
-pub struct ExecutablePayload(Box<ExecutionEnvironment>)
+pub struct ExecutablePayload(Box<ExecutionEnvironment>);
 pub struct UnlockablePayload(Box<ExecutionEnvironment>);
 pub struct PersistablePlayload(Box<ExecutionEnvironment>);
 

From bf5c4d3c89d926aab19e5ef3e2a643b138b1e72f Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 3 Sep 2022 23:28:09 +0900
Subject: [PATCH 1909/3199] save

---
 scheduler/src/lib.rs | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 1f21b1909aa200..ced6afcd61e83d 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1269,9 +1269,9 @@ impl ScheduleStage {
         runnable_queue: &mut TaskQueue,
         address_book: &mut AddressBook,
         from: &crossbeam_channel::Receiver<SchedulablePayload>,
-        from_exec: &crossbeam_channel::Receiver<Box<ExecutionEnvironment>>,
-        to_execute_substage: &crossbeam_channel::Sender<Box<ExecutionEnvironment>>,
-        maybe_to_next_stage: Option<&crossbeam_channel::Sender<Box<ExecutionEnvironment>>>, // assume nonblocking
+        to_execute_substage: &crossbeam_channel::Sender<ExecutablePayload>,
+        from_exec: &crossbeam_channel::Receiver<UnlockablePayload>,
+        maybe_to_next_stage: Option<&crossbeam_channel::Sender<PersistablePlayload>>, // assume nonblocking
     ) {
         let mut executing_queue_count = 0_usize;
         let mut contended_count = 0;
@@ -1446,8 +1446,8 @@ impl ScheduleStage {
         runnable_queue: &mut TaskQueue,
         address_book: &mut AddressBook,
         from: &crossbeam_channel::Receiver<SchedulablePayload>,
-        from_exec: &crossbeam_channel::Receiver<UnlockablePayload>,
         to_execute_substage: &crossbeam_channel::Sender<ExecutablePayload>,
+        from_execute_substage: &crossbeam_channel::Receiver<UnlockablePayload>,
         maybe_to_next_stage: Option<&crossbeam_channel::Sender<PersistablePlayload>>, // assume nonblocking
     ) {
         #[derive(Clone, Copy, Debug)]
@@ -1460,8 +1460,8 @@ impl ScheduleStage {
             runnable_queue,
             address_book,
             from,
-            from_exec,
             to_execute_substage,
+            from_execute_substage,
             maybe_to_next_stage,
         )
     }

From b020813c500cb1186ca545b3cd0eb93d3a7c735b Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 3 Sep 2022 23:28:22 +0900
Subject: [PATCH 1910/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index ced6afcd61e83d..d202030741591f 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1415,7 +1415,7 @@ impl ScheduleStage {
                     break;
                 } else {
                     if !empty_from_exec {
-                        let mut processed_execution_environment = from_exec.recv().unwrap();
+                        let mut processed_execution_environment = from_exec.recv().unwrap().0;
                         from_exec_len = from_exec_len.checked_sub(1).unwrap();
                         empty_from_exec = from_exec_len == 0;
                         executing_queue_count = executing_queue_count.checked_sub(1).unwrap();

From 7ed70c2a6ba3aa9535c08636ee94c1c0ada0e0c6 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 3 Sep 2022 23:28:45 +0900
Subject: [PATCH 1911/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index d202030741591f..8de8fd152811ac 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1392,7 +1392,7 @@ impl ScheduleStage {
                         &mut provisioning_tracker_count,
                     ) {
                         executing_queue_count = executing_queue_count.checked_add(1).unwrap();
-                        to_execute_substage.send(ee).unwrap();
+                        to_execute_substage.send(ExecutablePayload(ee)).unwrap();
                     } else {
                         break;
                     }

From 72e30aa729eea7f34ef07dd375d635764107d93d Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 3 Sep 2022 23:28:58 +0900
Subject: [PATCH 1912/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 8de8fd152811ac..6ebde8bd0de318 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1345,7 +1345,7 @@ impl ScheduleStage {
                            continue;
                        }
                    }
-                   let mut processed_execution_environment = maybe_from_exec.unwrap();
+                   let mut processed_execution_environment = maybe_from_exec.unwrap().0;
                     executing_queue_count = executing_queue_count.checked_sub(1).unwrap();
                     Self::commit_completed_execution(ast, &mut processed_execution_environment, address_book, &mut execute_clock, &mut provisioning_tracker_count);
                     to_next_stage.send(processed_execution_environment).unwrap();

From bfb9d0be7896492586a2637acc7abb2c1bbbd67e Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 3 Sep 2022 23:29:44 +0900
Subject: [PATCH 1913/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 6ebde8bd0de318..337b2cfbae1015 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1279,7 +1279,7 @@ impl ScheduleStage {
         let mut sequence_time = 0;
         let mut queue_clock = 0;
         let mut execute_clock = 0;
-        let (ee_sender, ee_receiver) = crossbeam_channel::unbounded::<Box<ExecutionEnvironment>>();
+        let (ee_sender, ee_receiver) = crossbeam_channel::unbounded::<PersistablePlayload>();
 
         let (to_next_stage, maybe_jon_handle) = if let Some(to_next_stage) = maybe_to_next_stage {
             (to_next_stage, None)

From 554ae0aa3ff5931c080e27d8d6d9d239f8f36dd1 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 3 Sep 2022 23:30:01 +0900
Subject: [PATCH 1914/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 337b2cfbae1015..07c6bd6986f098 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1426,7 +1426,7 @@ impl ScheduleStage {
                             &mut execute_clock,
                             &mut provisioning_tracker_count,
                         );
-                        to_next_stage.send(processed_execution_environment).unwrap();
+                        to_next_stage.send(PersistablePlayload(processed_execution_environment)).unwrap();
                     }
                     if !empty_from {
                         let task = from.recv().unwrap().0;

From 22abd5df773cc383a4126b60e2f389eee748c760 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 3 Sep 2022 23:30:19 +0900
Subject: [PATCH 1915/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 07c6bd6986f098..3b8c3749814d49 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1348,7 +1348,7 @@ impl ScheduleStage {
                    let mut processed_execution_environment = maybe_from_exec.unwrap().0;
                     executing_queue_count = executing_queue_count.checked_sub(1).unwrap();
                     Self::commit_completed_execution(ast, &mut processed_execution_environment, address_book, &mut execute_clock, &mut provisioning_tracker_count);
-                    to_next_stage.send(processed_execution_environment).unwrap();
+                    to_next_stage.send(PersistablePlayload(processed_execution_environment)).unwrap();
                }
                recv(from) -> maybe_from => {
                    if maybe_from.is_err() {

From 27847725f6d678e409ee6524d39d69dea234cbec Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 3 Sep 2022 23:30:49 +0900
Subject: [PATCH 1916/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 3b8c3749814d49..dc815896ee4dbf 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1292,7 +1292,7 @@ impl ScheduleStage {
                     unsafe impl NotAtScheduleThread for NotAtTopOfScheduleThread {}
                     let nast = NotAtTopOfScheduleThread;
 
-                    while let Ok(mut a) = ee_receiver.recv() {
+                    while let Ok(PersistablePlayload(mut a)) = ee_receiver.recv() {
                         assert!(a.task.lock_attempts_not_mut(nast).is_empty());
                         //assert!(a.task.sequence_time() != usize::max_value());
                         //let lock_attempts = std::mem::take(&mut a.lock_attempts);

From 004dfb618ef7ada155093d421671d243dabcbe5c Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 3 Sep 2022 23:31:12 +0900
Subject: [PATCH 1917/3199] save

---
 runtime/src/bank.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index a9062c71a89bf5..98b4bf2f0ab11d 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1200,7 +1200,7 @@ impl AbiExample for BuiltinPrograms {
 struct Scheduler {
     scheduler_thread_handle: Option<std::thread::JoinHandle<Result<()>>>,
     executing_thread_handle: Option<std::thread::JoinHandle<()>>,
-    transaction_sender: Option<crossbeam_channel::Sender<solana_scheduler::Schedulable>>,
+    transaction_sender: Option<crossbeam_channel::Sender<solana_scheduler::SchedulablePayload>>,
     preloader: Arc<solana_scheduler::Preloader>,
 }
 

From 0f80ca9430321de743095cce3ba0c50517b59e37 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 3 Sep 2022 23:31:37 +0900
Subject: [PATCH 1918/3199] save

---
 runtime/src/bank.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 98b4bf2f0ab11d..cb93369929bbc7 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1223,8 +1223,8 @@ impl Default for Scheduler {
                 &mut runnable_queue,
                 &mut address_book,
                 &transaction_receiver.clone(),
-                &completed_transaction_receiver,
                 &transaction_batch_sender,
+                &completed_transaction_receiver,
                 None,//&completed_transaction_receiver
             );
             Ok(())

From 919137f9d93fd17e34dacb0782552e3b8166ee68 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 3 Sep 2022 23:32:14 +0900
Subject: [PATCH 1919/3199] save

---
 runtime/src/bank.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index cb93369929bbc7..58ea8dd1cc390f 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1210,8 +1210,8 @@ impl Default for Scheduler {
         let mut address_book = solana_scheduler::AddressBook::default();
         let preloader = Arc::new(address_book.preloader());
         let (transaction_sender, transaction_receiver) = crossbeam_channel::unbounded();
+        let (ee_sender, ee_receiver) = crossbeam_channel::unbounded();
         let (completed_transaction_sender, completed_transaction_receiver) = crossbeam_channel::unbounded();
-        let (transaction_batch_sender, transaction_batch_receiver) = crossbeam_channel::unbounded();
         let executing_thread_handle = std::thread::Builder::new().name(format!("sol-exec-{}", 0)).spawn(move || {
         }).unwrap();
 
@@ -1223,7 +1223,7 @@ impl Default for Scheduler {
                 &mut runnable_queue,
                 &mut address_book,
                 &transaction_receiver.clone(),
-                &transaction_batch_sender,
+                &ee_sender,
                 &completed_transaction_receiver,
                 None,//&completed_transaction_receiver
             );

From a515e97cd89e7605a4dc488c574f82ed12ba965c Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 3 Sep 2022 23:34:26 +0900
Subject: [PATCH 1920/3199] save

---
 runtime/src/bank.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 58ea8dd1cc390f..6a0d387aef9930 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1211,7 +1211,7 @@ impl Default for Scheduler {
         let preloader = Arc::new(address_book.preloader());
         let (transaction_sender, transaction_receiver) = crossbeam_channel::unbounded();
         let (ee_sender, ee_receiver) = crossbeam_channel::unbounded();
-        let (completed_transaction_sender, completed_transaction_receiver) = crossbeam_channel::unbounded();
+        let (completed_ee_sender, completed_ee_receiver) = crossbeam_channel::unbounded();
         let executing_thread_handle = std::thread::Builder::new().name(format!("sol-exec-{}", 0)).spawn(move || {
         }).unwrap();
 
@@ -1224,7 +1224,7 @@ impl Default for Scheduler {
                 &mut address_book,
                 &transaction_receiver.clone(),
                 &ee_sender,
-                &completed_transaction_receiver,
+                &completed_ee_receiver,
                 None,//&completed_transaction_receiver
             );
             Ok(())

From d29cd9c70e4d94c7749702471ff21fda55a53848 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 3 Sep 2022 23:36:15 +0900
Subject: [PATCH 1921/3199] save

---
 runtime/src/bank.rs | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 6a0d387aef9930..e9a3d80b5c4f26 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -7896,7 +7896,10 @@ impl Bank {
     }
 
     pub fn wait_for_scheduler(&self) -> Result<()> {
-        let h = self.scheduler.write().unwrap().scheduler_thread_handle.take().unwrap();
+        let scheduler = self.scheduler.write().unwrap();
+        let transaction_sender = scheduler.transaction_sender.take().unwrap();
+        drop(transaction_sender);
+        let h = scheduler.scheduler_thread_handle.take().unwrap();
         h.join().unwrap()?;
 
         Ok(())

From 70fe9c38c795e754fb9ef9c71abbbcd0b9642890 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 3 Sep 2022 23:36:28 +0900
Subject: [PATCH 1922/3199] save

---
 runtime/src/bank.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index e9a3d80b5c4f26..aff745856922db 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -7897,7 +7897,7 @@ impl Bank {
 
     pub fn wait_for_scheduler(&self) -> Result<()> {
         let scheduler = self.scheduler.write().unwrap();
-        let transaction_sender = scheduler.transaction_sender.take().unwrap();
+        let mut transaction_sender = scheduler.transaction_sender.take().unwrap();
         drop(transaction_sender);
         let h = scheduler.scheduler_thread_handle.take().unwrap();
         h.join().unwrap()?;

From a404c09e777610a2c0c879b67cdd72af7f3d1593 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 3 Sep 2022 23:36:38 +0900
Subject: [PATCH 1923/3199] save

---
 runtime/src/bank.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index aff745856922db..baf8cd1b2b6999 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -7896,7 +7896,7 @@ impl Bank {
     }
 
     pub fn wait_for_scheduler(&self) -> Result<()> {
-        let scheduler = self.scheduler.write().unwrap();
+        let mut scheduler = self.scheduler.write().unwrap();
         let mut transaction_sender = scheduler.transaction_sender.take().unwrap();
         drop(transaction_sender);
         let h = scheduler.scheduler_thread_handle.take().unwrap();

From 3082274f5009130d6a01256bf521ad3a1203544b Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 3 Sep 2022 23:37:28 +0900
Subject: [PATCH 1924/3199] save

---
 runtime/src/bank.rs | 1 -
 1 file changed, 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index baf8cd1b2b6999..8cb86555e9398a 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1206,7 +1206,6 @@ struct Scheduler {
 
 impl Default for Scheduler {
     fn default() -> Self {
-        let mut runnable_queue = solana_scheduler::TaskQueue::default();
         let mut address_book = solana_scheduler::AddressBook::default();
         let preloader = Arc::new(address_book.preloader());
         let (transaction_sender, transaction_receiver) = crossbeam_channel::unbounded();

From 4e3030d6c653e15ff058422ce6ec10cd476e7e06 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 3 Sep 2022 23:41:08 +0900
Subject: [PATCH 1925/3199] save

---
 runtime/src/bank.rs | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 8cb86555e9398a..9f2a275e41cb5c 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1212,6 +1212,8 @@ impl Default for Scheduler {
         let (ee_sender, ee_receiver) = crossbeam_channel::unbounded();
         let (completed_ee_sender, completed_ee_receiver) = crossbeam_channel::unbounded();
         let executing_thread_handle = std::thread::Builder::new().name(format!("sol-exec-{}", 0)).spawn(move || {
+            while let Ok(a) = ee_receiver.recv() {
+            }
         }).unwrap();
 
         let scheduler_thread_handle = std::thread::Builder::new().name("solScheduler".to_string()).spawn(move || {

From f1387bc9088104ef5877f4e35342693ae2084459 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 3 Sep 2022 23:43:08 +0900
Subject: [PATCH 1926/3199] save

---
 runtime/src/bank.rs | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 9f2a275e41cb5c..285d62c6de1ca1 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1209,10 +1209,10 @@ impl Default for Scheduler {
         let mut address_book = solana_scheduler::AddressBook::default();
         let preloader = Arc::new(address_book.preloader());
         let (transaction_sender, transaction_receiver) = crossbeam_channel::unbounded();
-        let (ee_sender, ee_receiver) = crossbeam_channel::unbounded();
+        let (scheduled_ee_sender, scheduled_ee_receiver) = crossbeam_channel::unbounded();
         let (completed_ee_sender, completed_ee_receiver) = crossbeam_channel::unbounded();
         let executing_thread_handle = std::thread::Builder::new().name(format!("sol-exec-{}", 0)).spawn(move || {
-            while let Ok(a) = ee_receiver.recv() {
+            while let Ok(a) = scheduled_ee_receiver.recv() {
             }
         }).unwrap();
 
@@ -1224,7 +1224,7 @@ impl Default for Scheduler {
                 &mut runnable_queue,
                 &mut address_book,
                 &transaction_receiver.clone(),
-                &ee_sender,
+                &scheduled_ee_sender,
                 &completed_ee_receiver,
                 None,//&completed_transaction_receiver
             );

From 99ee72d5e08d9d61f6b6df70ac04d2021ff707d4 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 3 Sep 2022 23:43:20 +0900
Subject: [PATCH 1927/3199] save

---
 runtime/src/bank.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 285d62c6de1ca1..c1528eba3f7c04 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1223,7 +1223,7 @@ impl Default for Scheduler {
                 100,
                 &mut runnable_queue,
                 &mut address_book,
-                &transaction_receiver.clone(),
+                &transaction_receiver,
                 &scheduled_ee_sender,
                 &completed_ee_receiver,
                 None,//&completed_transaction_receiver

From bd4b076309ef54d8e174d3d209d89fd8f27fe778 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 3 Sep 2022 23:43:59 +0900
Subject: [PATCH 1928/3199] save

---
 runtime/src/bank.rs | 1 +
 1 file changed, 1 insertion(+)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index c1528eba3f7c04..bf0f6bf028ff63 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1213,6 +1213,7 @@ impl Default for Scheduler {
         let (completed_ee_sender, completed_ee_receiver) = crossbeam_channel::unbounded();
         let executing_thread_handle = std::thread::Builder::new().name(format!("sol-exec-{}", 0)).spawn(move || {
             while let Ok(a) = scheduled_ee_receiver.recv() {
+                completed_ee_sender.send(a).unwrap();
             }
         }).unwrap();
 

From 03c64d8d5085cfe5905559e46a58b51298b16566 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 3 Sep 2022 23:44:44 +0900
Subject: [PATCH 1929/3199] save

---
 runtime/src/bank.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index bf0f6bf028ff63..d425c029ed08b3 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1212,8 +1212,8 @@ impl Default for Scheduler {
         let (scheduled_ee_sender, scheduled_ee_receiver) = crossbeam_channel::unbounded();
         let (completed_ee_sender, completed_ee_receiver) = crossbeam_channel::unbounded();
         let executing_thread_handle = std::thread::Builder::new().name(format!("sol-exec-{}", 0)).spawn(move || {
-            while let Ok(a) = scheduled_ee_receiver.recv() {
-                completed_ee_sender.send(a).unwrap();
+            while let Ok(ExecutablePayload(a)) = scheduled_ee_receiver.recv() {
+                completed_ee_sender.send(UnlockablePayload(a)).unwrap();
             }
         }).unwrap();
 

From 7e17a1b0f4f42dc0b83031d3f84292d8f5016379 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 3 Sep 2022 23:45:19 +0900
Subject: [PATCH 1930/3199] save

---
 runtime/src/bank.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index d425c029ed08b3..a318da031d4d1e 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1212,8 +1212,8 @@ impl Default for Scheduler {
         let (scheduled_ee_sender, scheduled_ee_receiver) = crossbeam_channel::unbounded();
         let (completed_ee_sender, completed_ee_receiver) = crossbeam_channel::unbounded();
         let executing_thread_handle = std::thread::Builder::new().name(format!("sol-exec-{}", 0)).spawn(move || {
-            while let Ok(ExecutablePayload(a)) = scheduled_ee_receiver.recv() {
-                completed_ee_sender.send(UnlockablePayload(a)).unwrap();
+            while let Ok(solana_scheduler::ExecutablePayload(a)) = scheduled_ee_receiver.recv() {
+                completed_ee_sender.send(solana_scheduler::UnlockablePayload(a)).unwrap();
             }
         }).unwrap();
 

From c4fa486a4032043537119bf1c72bc72b83b0ef92 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 3 Sep 2022 23:45:37 +0900
Subject: [PATCH 1931/3199] save

---
 scheduler/src/lib.rs | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index dc815896ee4dbf..5a5546c019f6eb 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1467,10 +1467,10 @@ impl ScheduleStage {
     }
 }
 
-pub struct SchedulablePayload(TaskInQueue);
-pub struct ExecutablePayload(Box<ExecutionEnvironment>);
-pub struct UnlockablePayload(Box<ExecutionEnvironment>);
-pub struct PersistablePlayload(Box<ExecutionEnvironment>);
+pub struct SchedulablePayload(pub TaskInQueue);
+pub struct ExecutablePayload(pub Box<ExecutionEnvironment>);
+pub struct UnlockablePayload(pub Box<ExecutionEnvironment>);
+pub struct PersistablePlayload(pub Box<ExecutionEnvironment>);
 
 struct ExecuteStage {
     //bank: Bank,

From 4d651fc1b021cbddc2a0ed0ce2bf65899d04eb1a Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 3 Sep 2022 23:46:47 +0900
Subject: [PATCH 1932/3199] save

---
 runtime/src/bank.rs | 1 +
 1 file changed, 1 insertion(+)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index a318da031d4d1e..106c93024e74f9 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1231,6 +1231,7 @@ impl Default for Scheduler {
             );
             Ok(())
         }).unwrap();
+        drop(completed_ee_receiver);
 
         Self {
             scheduler_thread_handle: Some(scheduler_thread_handle),

From cfb8d10f69218529f25dbf646309da5ca2eb7af3 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 3 Sep 2022 23:46:59 +0900
Subject: [PATCH 1933/3199] save

---
 runtime/src/bank.rs | 1 -
 1 file changed, 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 106c93024e74f9..a318da031d4d1e 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1231,7 +1231,6 @@ impl Default for Scheduler {
             );
             Ok(())
         }).unwrap();
-        drop(completed_ee_receiver);
 
         Self {
             scheduler_thread_handle: Some(scheduler_thread_handle),

From 6b8436e5b6773efb5aa58febec5ff7839849d9b2 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 3 Sep 2022 23:47:24 +0900
Subject: [PATCH 1934/3199] save

---
 runtime/src/bank.rs | 1 +
 1 file changed, 1 insertion(+)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index a318da031d4d1e..a00adf13509120 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1229,6 +1229,7 @@ impl Default for Scheduler {
                 &completed_ee_receiver,
                 None,//&completed_transaction_receiver
             );
+            drop(completed_ee_receiver);
             Ok(())
         }).unwrap();
 

From 2a3637fd8ef8719cadb1b21958e091d16ca2c83c Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 3 Sep 2022 23:47:50 +0900
Subject: [PATCH 1935/3199] save

---
 runtime/src/bank.rs | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index a00adf13509120..15a76f8b2c0097 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1229,7 +1229,10 @@ impl Default for Scheduler {
                 &completed_ee_receiver,
                 None,//&completed_transaction_receiver
             );
+            drop(transaction_sender);
+            drop(scheduled_ee_sender);
             drop(completed_ee_receiver);
+
             Ok(())
         }).unwrap();
 

From 1f1e3293666b1760d3dcbbd69320dbeff4c689ad Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 3 Sep 2022 23:48:06 +0900
Subject: [PATCH 1936/3199] save

---
 runtime/src/bank.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 15a76f8b2c0097..549e359885729f 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1229,7 +1229,7 @@ impl Default for Scheduler {
                 &completed_ee_receiver,
                 None,//&completed_transaction_receiver
             );
-            drop(transaction_sender);
+            drop(transaction_receiver);
             drop(scheduled_ee_sender);
             drop(completed_ee_receiver);
 

From 77b9a98081d29e029934a299681fe3f5ad2d4a54 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 3 Sep 2022 23:49:17 +0900
Subject: [PATCH 1937/3199] save

---
 runtime/src/bank.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 549e359885729f..391a99c6972f36 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1227,7 +1227,7 @@ impl Default for Scheduler {
                 &transaction_receiver,
                 &scheduled_ee_sender,
                 &completed_ee_receiver,
-                None,//&completed_transaction_receiver
+                None,
             );
             drop(transaction_receiver);
             drop(scheduled_ee_sender);

From b91961ede5abe5d9fa67e9fd567f01af2a0997f7 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 3 Sep 2022 23:49:37 +0900
Subject: [PATCH 1938/3199] save

---
 runtime/src/bank.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 391a99c6972f36..85652a69d7e20e 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1212,8 +1212,8 @@ impl Default for Scheduler {
         let (scheduled_ee_sender, scheduled_ee_receiver) = crossbeam_channel::unbounded();
         let (completed_ee_sender, completed_ee_receiver) = crossbeam_channel::unbounded();
         let executing_thread_handle = std::thread::Builder::new().name(format!("sol-exec-{}", 0)).spawn(move || {
-            while let Ok(solana_scheduler::ExecutablePayload(a)) = scheduled_ee_receiver.recv() {
-                completed_ee_sender.send(solana_scheduler::UnlockablePayload(a)).unwrap();
+            while let Ok(solana_scheduler::ExecutablePayload(ee)) = scheduled_ee_receiver.recv() {
+                completed_ee_sender.send(solana_scheduler::UnlockablePayload(ee)).unwrap();
             }
         }).unwrap();
 

From 7a2af58260c34af50c888d5a9d045176695a865a Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 3 Sep 2022 23:53:00 +0900
Subject: [PATCH 1939/3199] save

---
 scheduler/src/lib.rs | 18 +++++++++---------
 1 file changed, 9 insertions(+), 9 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 5a5546c019f6eb..495cf25ee4230c 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1268,7 +1268,7 @@ impl ScheduleStage {
         max_executing_queue_count: usize,
         runnable_queue: &mut TaskQueue,
         address_book: &mut AddressBook,
-        from: &crossbeam_channel::Receiver<SchedulablePayload>,
+        from_prev: &crossbeam_channel::Receiver<SchedulablePayload>,
         to_execute_substage: &crossbeam_channel::Sender<ExecutablePayload>,
         from_exec: &crossbeam_channel::Receiver<UnlockablePayload>,
         maybe_to_next_stage: Option<&crossbeam_channel::Sender<PersistablePlayload>>, // assume nonblocking
@@ -1350,9 +1350,9 @@ impl ScheduleStage {
                     Self::commit_completed_execution(ast, &mut processed_execution_environment, address_book, &mut execute_clock, &mut provisioning_tracker_count);
                     to_next_stage.send(PersistablePlayload(processed_execution_environment)).unwrap();
                }
-               recv(from) -> maybe_from => {
+               recv(from_prev) -> maybe_from => {
                    if maybe_from.is_err() {
-                       assert_eq!(from.len(), 0);
+                       assert_eq!(from_prev.len(), 0);
                        from_disconnected = true;
                        if from_exec_disconnected {
                            break;
@@ -1373,10 +1373,10 @@ impl ScheduleStage {
                 while (executing_queue_count + provisioning_tracker_count)
                     < max_executing_queue_count
                 {
-                    trace!("schedule_once (from: {}, to: {}, runnnable: {}, contended: {}, (immediate+provisional)/max: ({}+{})/{}) active from contended: {} stuck: {}!", from.len(), to_execute_substage.len(), runnable_queue.task_count(), contended_count, executing_queue_count, provisioning_tracker_count, max_executing_queue_count, address_book.uncontended_task_ids.len(), address_book.stuck_tasks.len());
+                    trace!("schedule_once (from: {}, to: {}, runnnable: {}, contended: {}, (immediate+provisional)/max: ({}+{})/{}) active from contended: {} stuck: {}!", from_prev.len(), to_execute_substage.len(), runnable_queue.task_count(), contended_count, executing_queue_count, provisioning_tracker_count, max_executing_queue_count, address_book.uncontended_task_ids.len(), address_book.stuck_tasks.len());
                     if start.elapsed() > std::time::Duration::from_millis(1000) {
                         start = std::time::Instant::now();
-                        info!("schedule_once (from: {}, to: {}, runnnable: {}, contended: {}, (immediate+provisional)/max: ({}+{})/{}) active from contended: {} stuck: {}!", from.len(), to_execute_substage.len(), runnable_queue.task_count(), contended_count, executing_queue_count, provisioning_tracker_count, max_executing_queue_count, address_book.uncontended_task_ids.len(), address_book.stuck_tasks.len());
+                        info!("schedule_once (from: {}, to: {}, runnnable: {}, contended: {}, (immediate+provisional)/max: ({}+{})/{}) active from contended: {} stuck: {}!", from_prev.len(), to_execute_substage.len(), runnable_queue.task_count(), contended_count, executing_queue_count, provisioning_tracker_count, max_executing_queue_count, address_book.uncontended_task_ids.len(), address_book.stuck_tasks.len());
                     }
                     let prefer_immediate = provisioning_tracker_count / 4 > executing_queue_count;
                     if let Some(ee) = Self::schedule_next_execution(
@@ -1400,10 +1400,10 @@ impl ScheduleStage {
                 //break;
                 if first_iteration {
                     first_iteration = false;
-                    (from_len, from_exec_len) = (from.len(), from_exec.len());
+                    (from_len, from_exec_len) = (from_prev.len(), from_exec.len());
                 } else {
                     if empty_from {
-                        from_len = from.len();
+                        from_len = from_prev.len();
                     }
                     if empty_from_exec {
                         from_exec_len = from_exec.len();
@@ -1429,7 +1429,7 @@ impl ScheduleStage {
                         to_next_stage.send(PersistablePlayload(processed_execution_environment)).unwrap();
                     }
                     if !empty_from {
-                        let task = from.recv().unwrap().0;
+                        let task = from_prev.recv().unwrap().0;
                         from_len = from_len.checked_sub(1).unwrap();
                         empty_from = from_len == 0;
                         Self::register_runnable_task(task, runnable_queue, &mut sequence_time);
@@ -1438,7 +1438,7 @@ impl ScheduleStage {
             }
         }
         info!("run finished...");
-        info!("schedule_once (from: {}, to: {}, runnnable: {}, contended: {}, (immediate+provisional)/max: ({}+{})/{}) active from contended: {} stuck: {}!", from.len(), to_execute_substage.len(), runnable_queue.task_count(), contended_count, executing_queue_count, provisioning_tracker_count, max_executing_queue_count, address_book.uncontended_task_ids.len(), address_book.stuck_tasks.len());
+        info!("schedule_once (from: {}, to: {}, runnnable: {}, contended: {}, (immediate+provisional)/max: ({}+{})/{}) active from contended: {} stuck: {}!", from_prev.len(), to_execute_substage.len(), runnable_queue.task_count(), contended_count, executing_queue_count, provisioning_tracker_count, max_executing_queue_count, address_book.uncontended_task_ids.len(), address_book.stuck_tasks.len());
     }
 
     pub fn run(

From c354e43e8ececdfc9f6cb0cf431a59f324aef3ab Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 3 Sep 2022 23:57:38 +0900
Subject: [PATCH 1940/3199] save

---
 runtime/src/bank.rs | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 85652a69d7e20e..f312c77d502cf9 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -7905,6 +7905,8 @@ impl Bank {
         let mut scheduler = self.scheduler.write().unwrap();
         let mut transaction_sender = scheduler.transaction_sender.take().unwrap();
         drop(transaction_sender);
+        let h = scheduler.executing_thread_handle.take().unwrap();
+        h.join().unwrap()?;
         let h = scheduler.scheduler_thread_handle.take().unwrap();
         h.join().unwrap()?;
 

From 9cc67bea9aaeaf7096ba7e9f1731acbe2bace7cd Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 3 Sep 2022 23:57:57 +0900
Subject: [PATCH 1941/3199] save

---
 runtime/src/bank.rs | 1 +
 1 file changed, 1 insertion(+)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index f312c77d502cf9..e4848e3c8859db 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1215,6 +1215,7 @@ impl Default for Scheduler {
             while let Ok(solana_scheduler::ExecutablePayload(ee)) = scheduled_ee_receiver.recv() {
                 completed_ee_sender.send(solana_scheduler::UnlockablePayload(ee)).unwrap();
             }
+            Ok(())
         }).unwrap();
 
         let scheduler_thread_handle = std::thread::Builder::new().name("solScheduler".to_string()).spawn(move || {

From 303d8cafe332f6f3d5616505adfab696523df7f5 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 3 Sep 2022 23:58:16 +0900
Subject: [PATCH 1942/3199] save

---
 runtime/src/bank.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index e4848e3c8859db..cfab722c846a9a 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1199,7 +1199,7 @@ impl AbiExample for BuiltinPrograms {
 #[derive(Debug)]
 struct Scheduler {
     scheduler_thread_handle: Option<std::thread::JoinHandle<Result<()>>>,
-    executing_thread_handle: Option<std::thread::JoinHandle<()>>,
+    executing_thread_handle: Option<std::thread::JoinHandle<Result<()>>>,
     transaction_sender: Option<crossbeam_channel::Sender<solana_scheduler::SchedulablePayload>>,
     preloader: Arc<solana_scheduler::Preloader>,
 }

From 24356ddf90e14f6bb0f87f44f4125ebc6af962a0 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 4 Sep 2022 00:06:20 +0900
Subject: [PATCH 1943/3199] save

---
 scheduler/src/lib.rs | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 495cf25ee4230c..c625ce557cd9f5 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1354,7 +1354,8 @@ impl ScheduleStage {
                    if maybe_from.is_err() {
                        assert_eq!(from_prev.len(), 0);
                        from_disconnected = true;
-                       if from_exec_disconnected {
+                       let finished = runnable_queue.task_count() + contended_count + executing_queue_count + provisioning_tracker_count == 0;
+                       if from_exec_disconnected || finished {
                            break;
                        } else {
                            continue;

From 654570f699caf559e2fbe26b8c2aea14d708c52d Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 4 Sep 2022 00:19:26 +0900
Subject: [PATCH 1944/3199] save

---
 runtime/src/bank.rs                     | 15 +++++++++++++++
 transaction-scheduler-bench/src/main.rs |  2 +-
 2 files changed, 16 insertions(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index cfab722c846a9a..a5584ba17c9171 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1204,6 +1204,11 @@ struct Scheduler {
     preloader: Arc<solana_scheduler::Preloader>,
 }
 
+impl Scheduler {
+    fn schedule(&self, sani: &SanitizedTransaction) {
+    }
+}
+
 impl Default for Scheduler {
     fn default() -> Self {
         let mut address_book = solana_scheduler::AddressBook::default();
@@ -4560,6 +4565,16 @@ impl Bank {
         }
     }
 
+    pub fn schedule_transaction_batch_to_be_commited(
+        &self,
+        batch: &TransactionBatch,
+    ) {
+        let scheduler = self.scheduler.read().unwrap();
+        for st in batch.sanitized_transactions() {
+            scheduler.schedule(st);
+        }
+    }
+
     #[allow(clippy::type_complexity)]
     pub fn load_and_execute_transactions(
         &self,
diff --git a/transaction-scheduler-bench/src/main.rs b/transaction-scheduler-bench/src/main.rs
index 6473ffad6a135f..4dc5f65c3197ce 100644
--- a/transaction-scheduler-bench/src/main.rs
+++ b/transaction-scheduler-bench/src/main.rs
@@ -139,8 +139,8 @@ fn spawn_unified_scheduler(
             &mut runnable_queue,
             &mut address_book,
             &packet_batch_receiver.clone(),
-            &completed_transaction_receiver,
             &transaction_batch_senders[0],
+            &completed_transaction_receiver,
             None,//&completed_transaction_receiver
         );
     }).unwrap()

From 6f87bb73d6f59168c931742d644cded028ee8c3a Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 4 Sep 2022 00:20:25 +0900
Subject: [PATCH 1945/3199] save

---
 runtime/src/bank.rs | 1 +
 1 file changed, 1 insertion(+)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index a5584ba17c9171..89f732db0680d8 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1206,6 +1206,7 @@ struct Scheduler {
 
 impl Scheduler {
     fn schedule(&self, sani: &SanitizedTransaction) {
+        self.transaction_sender.send(solana_scheduler::SchedulablePayload(sani)).unwrap();
     }
 }
 

From a3aba9d4f22a007dae64e341e66bfb690ca57e8a Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 4 Sep 2022 00:21:30 +0900
Subject: [PATCH 1946/3199] save

---
 runtime/src/bank.rs | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 89f732db0680d8..31f1d45be216a7 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1206,6 +1206,11 @@ struct Scheduler {
 
 impl Scheduler {
     fn schedule(&self, sani: &SanitizedTransaction) {
+        #[derive(Clone, Copy, Debug)]
+        struct NotAtTopOfScheduleThread;
+        unsafe impl solana_scheduler::NotAtScheduleThread for NotAtTopOfScheduleThread {}
+        let nast = NotAtTopOfScheduleThread;
+
         self.transaction_sender.send(solana_scheduler::SchedulablePayload(sani)).unwrap();
     }
 }

From c9d74feafc516b3feb6ba4074897a10df0ab785c Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 4 Sep 2022 00:22:16 +0900
Subject: [PATCH 1947/3199] save

---
 runtime/src/bank.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 31f1d45be216a7..ed311cf6a423d3 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1210,8 +1210,8 @@ impl Scheduler {
         struct NotAtTopOfScheduleThread;
         unsafe impl solana_scheduler::NotAtScheduleThread for NotAtTopOfScheduleThread {}
         let nast = NotAtTopOfScheduleThread;
-
-        self.transaction_sender.send(solana_scheduler::SchedulablePayload(sani)).unwrap();
+        let t = solana_scheduler::Task::new_for_queue(nast, 0, sani);
+        self.transaction_sender.send(solana_scheduler::SchedulablePayload(t)).unwrap();
     }
 }
 

From 1bad1492c5ce1bd2e2f3419d7b59e42a7f51d35a Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 4 Sep 2022 00:22:36 +0900
Subject: [PATCH 1948/3199] save

---
 runtime/src/bank.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index ed311cf6a423d3..04b98ece837f97 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1211,7 +1211,7 @@ impl Scheduler {
         unsafe impl solana_scheduler::NotAtScheduleThread for NotAtTopOfScheduleThread {}
         let nast = NotAtTopOfScheduleThread;
         let t = solana_scheduler::Task::new_for_queue(nast, 0, sani);
-        self.transaction_sender.send(solana_scheduler::SchedulablePayload(t)).unwrap();
+        self.transaction_sender.unwrap().send(solana_scheduler::SchedulablePayload(t)).unwrap();
     }
 }
 

From c2f094e38b7a697d353b07ad0a97bae7b9e1e9ee Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 4 Sep 2022 13:35:10 +0900
Subject: [PATCH 1949/3199] save

---
 runtime/src/bank.rs | 16 ++++++++++++++--
 1 file changed, 14 insertions(+), 2 deletions(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 04b98ece837f97..40f4f6a646f066 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1205,12 +1205,24 @@ struct Scheduler {
 }
 
 impl Scheduler {
-    fn schedule(&self, sani: &SanitizedTransaction) {
+    fn schedule(&self, sanitized_tx: &SanitizedTransaction) {
         #[derive(Clone, Copy, Debug)]
         struct NotAtTopOfScheduleThread;
         unsafe impl solana_scheduler::NotAtScheduleThread for NotAtTopOfScheduleThread {}
         let nast = NotAtTopOfScheduleThread;
-        let t = solana_scheduler::Task::new_for_queue(nast, 0, sani);
+
+        let locks = sanitized_tx.get_account_locks_unchecked();
+        let writable_lock_iter = locks
+            .writable
+            .iter()
+            .map(|address| solana_scheduler::LockAttempt::new(preloader.load(**address), solana_scheduler::RequestedUsage::Writable));
+        let readonly_lock_iter = locks
+            .readonly
+            .iter()
+            .map(|address| solana_scheduler::LockAttempt::new(preloader.load(**address), solana_scheduler::RequestedUsage::Readonly));
+        let locks = writable_lock_iter.chain(readonly_lock_iter).collect::<Vec<_>>();
+
+        let t = solana_scheduler::Task::new_for_queue(nast, 0, sanitized_txs);
         self.transaction_sender.unwrap().send(solana_scheduler::SchedulablePayload(t)).unwrap();
     }
 }

From f9240fddb92d22dabaf29d51970d5c0f1dc4f8a7 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 4 Sep 2022 13:35:44 +0900
Subject: [PATCH 1950/3199] save

---
 runtime/src/bank.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 40f4f6a646f066..c2697ec223b05f 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1215,11 +1215,11 @@ impl Scheduler {
         let writable_lock_iter = locks
             .writable
             .iter()
-            .map(|address| solana_scheduler::LockAttempt::new(preloader.load(**address), solana_scheduler::RequestedUsage::Writable));
+            .map(|address| solana_scheduler::LockAttempt::new(self.preloader.load(**address), solana_scheduler::RequestedUsage::Writable));
         let readonly_lock_iter = locks
             .readonly
             .iter()
-            .map(|address| solana_scheduler::LockAttempt::new(preloader.load(**address), solana_scheduler::RequestedUsage::Readonly));
+            .map(|address| solana_scheduler::LockAttempt::new(self.preloader.load(**address), solana_scheduler::RequestedUsage::Readonly));
         let locks = writable_lock_iter.chain(readonly_lock_iter).collect::<Vec<_>>();
 
         let t = solana_scheduler::Task::new_for_queue(nast, 0, sanitized_txs);

From 34f187ea89c21a92559c87369a0da6e56eecbe00 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 4 Sep 2022 13:35:55 +0900
Subject: [PATCH 1951/3199] save

---
 runtime/src/bank.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index c2697ec223b05f..9a15bcaa8cbfd0 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1222,7 +1222,7 @@ impl Scheduler {
             .map(|address| solana_scheduler::LockAttempt::new(self.preloader.load(**address), solana_scheduler::RequestedUsage::Readonly));
         let locks = writable_lock_iter.chain(readonly_lock_iter).collect::<Vec<_>>();
 
-        let t = solana_scheduler::Task::new_for_queue(nast, 0, sanitized_txs);
+        let t = solana_scheduler::Task::new_for_queue(nast, 0, sanitized_tx);
         self.transaction_sender.unwrap().send(solana_scheduler::SchedulablePayload(t)).unwrap();
     }
 }

From 35d92b866cf030a4c600a6477a72832df4efb7b2 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 4 Sep 2022 13:36:12 +0900
Subject: [PATCH 1952/3199] save

---
 runtime/src/bank.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 9a15bcaa8cbfd0..c20106625a34da 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1205,7 +1205,7 @@ struct Scheduler {
 }
 
 impl Scheduler {
-    fn schedule(&self, sanitized_tx: &SanitizedTransaction) {
+    fn schedule(&self, sanitized_tx: SanitizedTransaction) {
         #[derive(Clone, Copy, Debug)]
         struct NotAtTopOfScheduleThread;
         unsafe impl solana_scheduler::NotAtScheduleThread for NotAtTopOfScheduleThread {}

From 497132d7cc0e9b5ed5c1cea2035c90c320884307 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 4 Sep 2022 13:36:29 +0900
Subject: [PATCH 1953/3199] save

---
 runtime/src/bank.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index c20106625a34da..ec44636440f953 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1222,7 +1222,7 @@ impl Scheduler {
             .map(|address| solana_scheduler::LockAttempt::new(self.preloader.load(**address), solana_scheduler::RequestedUsage::Readonly));
         let locks = writable_lock_iter.chain(readonly_lock_iter).collect::<Vec<_>>();
 
-        let t = solana_scheduler::Task::new_for_queue(nast, 0, sanitized_tx);
+        let t = solana_scheduler::Task::new_for_queue(nast, 0, (sanitized_tx, locks));
         self.transaction_sender.unwrap().send(solana_scheduler::SchedulablePayload(t)).unwrap();
     }
 }

From 14f1d801b5139af8036f60e7eaa154259b177a72 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 4 Sep 2022 13:36:46 +0900
Subject: [PATCH 1954/3199] save

---
 runtime/src/bank.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index ec44636440f953..b09c9e5bb64e7d 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1222,7 +1222,7 @@ impl Scheduler {
             .map(|address| solana_scheduler::LockAttempt::new(self.preloader.load(**address), solana_scheduler::RequestedUsage::Readonly));
         let locks = writable_lock_iter.chain(readonly_lock_iter).collect::<Vec<_>>();
 
-        let t = solana_scheduler::Task::new_for_queue(nast, 0, (sanitized_tx, locks));
+        let t = solana_scheduler::Task::new_for_queue(nast, 0, (sanitized_tx.clone(), locks));
         self.transaction_sender.unwrap().send(solana_scheduler::SchedulablePayload(t)).unwrap();
     }
 }

From bf4be177e35a83beaaabf366478927d4f913ddcd Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 4 Sep 2022 13:36:59 +0900
Subject: [PATCH 1955/3199] save

---
 runtime/src/bank.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index b09c9e5bb64e7d..6b24abeda94f7e 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1205,7 +1205,7 @@ struct Scheduler {
 }
 
 impl Scheduler {
-    fn schedule(&self, sanitized_tx: SanitizedTransaction) {
+    fn schedule(&self, sanitized_tx: &SanitizedTransaction) {
         #[derive(Clone, Copy, Debug)]
         struct NotAtTopOfScheduleThread;
         unsafe impl solana_scheduler::NotAtScheduleThread for NotAtTopOfScheduleThread {}

From be3cb410d35c272b485219f991109ca97c2da5ec Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 4 Sep 2022 13:37:26 +0900
Subject: [PATCH 1956/3199] save

---
 runtime/src/bank.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 6b24abeda94f7e..54c33551723051 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1223,7 +1223,7 @@ impl Scheduler {
         let locks = writable_lock_iter.chain(readonly_lock_iter).collect::<Vec<_>>();
 
         let t = solana_scheduler::Task::new_for_queue(nast, 0, (sanitized_tx.clone(), locks));
-        self.transaction_sender.unwrap().send(solana_scheduler::SchedulablePayload(t)).unwrap();
+        self.transaction_sender.as_ref().unwrap().send(solana_scheduler::SchedulablePayload(t)).unwrap();
     }
 }
 

From aedbb571a2d2bceb00ebae4f853f09fb16cb563a Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 4 Sep 2022 13:39:05 +0900
Subject: [PATCH 1957/3199] save

---
 scheduler/src/lib.rs | 1 +
 1 file changed, 1 insertion(+)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index c625ce557cd9f5..f904e3ab478520 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1438,6 +1438,7 @@ impl ScheduleStage {
                 }
             }
         }
+        drop(to_next_stage);
         info!("run finished...");
         info!("schedule_once (from: {}, to: {}, runnnable: {}, contended: {}, (immediate+provisional)/max: ({}+{})/{}) active from contended: {} stuck: {}!", from_prev.len(), to_execute_substage.len(), runnable_queue.task_count(), contended_count, executing_queue_count, provisioning_tracker_count, max_executing_queue_count, address_book.uncontended_task_ids.len(), address_book.stuck_tasks.len());
     }

From 4e690c23a5b4b529873791eac7d09e1aa89e6f65 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 4 Sep 2022 13:41:58 +0900
Subject: [PATCH 1958/3199] save

---
 scheduler/src/lib.rs | 1 +
 1 file changed, 1 insertion(+)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index f904e3ab478520..6848db772ae6cc 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1439,6 +1439,7 @@ impl ScheduleStage {
             }
         }
         drop(to_next_stage);
+        drop(ee_sender);
         info!("run finished...");
         info!("schedule_once (from: {}, to: {}, runnnable: {}, contended: {}, (immediate+provisional)/max: ({}+{})/{}) active from contended: {} stuck: {}!", from_prev.len(), to_execute_substage.len(), runnable_queue.task_count(), contended_count, executing_queue_count, provisioning_tracker_count, max_executing_queue_count, address_book.uncontended_task_ids.len(), address_book.stuck_tasks.len());
     }

From bf2ff3704299b73692377e598efb0a5694cabc83 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 4 Sep 2022 13:42:06 +0900
Subject: [PATCH 1959/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 6848db772ae6cc..0ae41720c47c2f 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1438,8 +1438,8 @@ impl ScheduleStage {
                 }
             }
         }
-        drop(to_next_stage);
         drop(ee_sender);
+        drop(to_next_stage);
         info!("run finished...");
         info!("schedule_once (from: {}, to: {}, runnnable: {}, contended: {}, (immediate+provisional)/max: ({}+{})/{}) active from contended: {} stuck: {}!", from_prev.len(), to_execute_substage.len(), runnable_queue.task_count(), contended_count, executing_queue_count, provisioning_tracker_count, max_executing_queue_count, address_book.uncontended_task_ids.len(), address_book.stuck_tasks.len());
     }

From 48c2f1135d753f94f500df09df8f718d131d109a Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 4 Sep 2022 13:42:22 +0900
Subject: [PATCH 1960/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 0ae41720c47c2f..6848db772ae6cc 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1438,8 +1438,8 @@ impl ScheduleStage {
                 }
             }
         }
-        drop(ee_sender);
         drop(to_next_stage);
+        drop(ee_sender);
         info!("run finished...");
         info!("schedule_once (from: {}, to: {}, runnnable: {}, contended: {}, (immediate+provisional)/max: ({}+{})/{}) active from contended: {} stuck: {}!", from_prev.len(), to_execute_substage.len(), runnable_queue.task_count(), contended_count, executing_queue_count, provisioning_tracker_count, max_executing_queue_count, address_book.uncontended_task_ids.len(), address_book.stuck_tasks.len());
     }

From 8c65d154b355830adf5be4d4a4db63bd18bed783 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 4 Sep 2022 13:42:55 +0900
Subject: [PATCH 1961/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 6848db772ae6cc..1e613370604986 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1286,7 +1286,7 @@ impl ScheduleStage {
         } else {
             let h = std::thread::Builder::new()
                 .name("sol-reaper".to_string())
-                .spawn(move || {
+                .spawn(|| {
                     #[derive(Clone, Copy, Debug)]
                     struct NotAtTopOfScheduleThread;
                     unsafe impl NotAtScheduleThread for NotAtTopOfScheduleThread {}

From 6bf51e1759cbc0af522018acf389acc557e0d6d9 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 4 Sep 2022 13:43:07 +0900
Subject: [PATCH 1962/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 1e613370604986..87598d241b14b4 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1292,7 +1292,7 @@ impl ScheduleStage {
                     unsafe impl NotAtScheduleThread for NotAtTopOfScheduleThread {}
                     let nast = NotAtTopOfScheduleThread;
 
-                    while let Ok(PersistablePlayload(mut a)) = ee_receiver.recv() {
+                    while let Ok(PersistablePlayload(mut a)) = &ee_receiver.recv() {
                         assert!(a.task.lock_attempts_not_mut(nast).is_empty());
                         //assert!(a.task.sequence_time() != usize::max_value());
                         //let lock_attempts = std::mem::take(&mut a.lock_attempts);

From 9abb4c70e4b9d644beaf8df0e5e2bcf0e5cf678f Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 4 Sep 2022 13:43:13 +0900
Subject: [PATCH 1963/3199] save

---
 scheduler/src/lib.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 87598d241b14b4..6848db772ae6cc 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1286,13 +1286,13 @@ impl ScheduleStage {
         } else {
             let h = std::thread::Builder::new()
                 .name("sol-reaper".to_string())
-                .spawn(|| {
+                .spawn(move || {
                     #[derive(Clone, Copy, Debug)]
                     struct NotAtTopOfScheduleThread;
                     unsafe impl NotAtScheduleThread for NotAtTopOfScheduleThread {}
                     let nast = NotAtTopOfScheduleThread;
 
-                    while let Ok(PersistablePlayload(mut a)) = &ee_receiver.recv() {
+                    while let Ok(PersistablePlayload(mut a)) = ee_receiver.recv() {
                         assert!(a.task.lock_attempts_not_mut(nast).is_empty());
                         //assert!(a.task.sequence_time() != usize::max_value());
                         //let lock_attempts = std::mem::take(&mut a.lock_attempts);

From 397b9921b90a4d198b416c7b6aab30ffa78b62f0 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 4 Sep 2022 13:44:29 +0900
Subject: [PATCH 1964/3199] save

---
 scheduler/src/lib.rs | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 6848db772ae6cc..be6cdbee3e3448 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1281,7 +1281,7 @@ impl ScheduleStage {
         let mut execute_clock = 0;
         let (ee_sender, ee_receiver) = crossbeam_channel::unbounded::<PersistablePlayload>();
 
-        let (to_next_stage, maybe_jon_handle) = if let Some(to_next_stage) = maybe_to_next_stage {
+        let (to_next_stage, maybe_reaper_thread_handle) = if let Some(to_next_stage) = maybe_to_next_stage {
             (to_next_stage, None)
         } else {
             let h = std::thread::Builder::new()
@@ -1440,6 +1440,9 @@ impl ScheduleStage {
         }
         drop(to_next_stage);
         drop(ee_sender);
+        if let Some(h) = maybe_reaper_thread_handle {
+            h.join().unwrap()
+        }
         info!("run finished...");
         info!("schedule_once (from: {}, to: {}, runnnable: {}, contended: {}, (immediate+provisional)/max: ({}+{})/{}) active from contended: {} stuck: {}!", from_prev.len(), to_execute_substage.len(), runnable_queue.task_count(), contended_count, executing_queue_count, provisioning_tracker_count, max_executing_queue_count, address_book.uncontended_task_ids.len(), address_book.stuck_tasks.len());
     }

From 4f9efb31674f74327784c8d0e094212d70267ed3 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 4 Sep 2022 13:44:58 +0900
Subject: [PATCH 1965/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index be6cdbee3e3448..4329fbefcf600e 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1441,7 +1441,7 @@ impl ScheduleStage {
         drop(to_next_stage);
         drop(ee_sender);
         if let Some(h) = maybe_reaper_thread_handle {
-            h.join().unwrap()
+            h.join().unwrap().unwrap();
         }
         info!("run finished...");
         info!("schedule_once (from: {}, to: {}, runnnable: {}, contended: {}, (immediate+provisional)/max: ({}+{})/{}) active from contended: {} stuck: {}!", from_prev.len(), to_execute_substage.len(), runnable_queue.task_count(), contended_count, executing_queue_count, provisioning_tracker_count, max_executing_queue_count, address_book.uncontended_task_ids.len(), address_book.stuck_tasks.len());

From f3403762776477ef0c1603b75f4b2962f0ddfc10 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 4 Sep 2022 13:45:17 +0900
Subject: [PATCH 1966/3199] save

---
 scheduler/src/lib.rs | 1 +
 1 file changed, 1 insertion(+)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 4329fbefcf600e..458e63f960ff1d 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1300,6 +1300,7 @@ impl ScheduleStage {
                         //TaskInQueue::get_mut(&mut a.task).unwrap();
                     }
                     assert_eq!(ee_receiver.len(), 0);
+                    Ok(())
                 })
                 .unwrap();
 

From 3c07b1c309fc4f115adbd514147cd17edcc508bc Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 4 Sep 2022 13:46:08 +0900
Subject: [PATCH 1967/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 458e63f960ff1d..fc89c36711839b 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1284,7 +1284,7 @@ impl ScheduleStage {
         let (to_next_stage, maybe_reaper_thread_handle) = if let Some(to_next_stage) = maybe_to_next_stage {
             (to_next_stage, None)
         } else {
-            let h = std::thread::Builder::new()
+            let h: JoinHandle<Result<(), ()> = std::thread::Builder::new()
                 .name("sol-reaper".to_string())
                 .spawn(move || {
                     #[derive(Clone, Copy, Debug)]

From 78d344a8734b510f8685bf4af12f3f9fe9be0966 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 4 Sep 2022 13:46:27 +0900
Subject: [PATCH 1968/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index fc89c36711839b..fa7cb0c32bb749 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1284,7 +1284,7 @@ impl ScheduleStage {
         let (to_next_stage, maybe_reaper_thread_handle) = if let Some(to_next_stage) = maybe_to_next_stage {
             (to_next_stage, None)
         } else {
-            let h: JoinHandle<Result<(), ()> = std::thread::Builder::new()
+            let h: JoinHandle::<Result<(), ()> = std::thread::Builder::new()
                 .name("sol-reaper".to_string())
                 .spawn(move || {
                     #[derive(Clone, Copy, Debug)]

From cae8be276656b452d2ab07cd93804aead9367c98 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 4 Sep 2022 13:46:38 +0900
Subject: [PATCH 1969/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index fa7cb0c32bb749..835d6036371ba6 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1284,7 +1284,7 @@ impl ScheduleStage {
         let (to_next_stage, maybe_reaper_thread_handle) = if let Some(to_next_stage) = maybe_to_next_stage {
             (to_next_stage, None)
         } else {
-            let h: JoinHandle::<Result<(), ()> = std::thread::Builder::new()
+            let h: JoinHandle::<Result<(), ()>> = std::thread::Builder::new()
                 .name("sol-reaper".to_string())
                 .spawn(move || {
                     #[derive(Clone, Copy, Debug)]

From d626167c3e213ff73172796fef4446ef3a19fb05 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 4 Sep 2022 13:46:47 +0900
Subject: [PATCH 1970/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 835d6036371ba6..72bd7372f91e75 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1284,7 +1284,7 @@ impl ScheduleStage {
         let (to_next_stage, maybe_reaper_thread_handle) = if let Some(to_next_stage) = maybe_to_next_stage {
             (to_next_stage, None)
         } else {
-            let h: JoinHandle::<Result<(), ()>> = std::thread::Builder::new()
+            let h: std::thread::JoinHandle::<Result<(), ()>> = std::thread::Builder::new()
                 .name("sol-reaper".to_string())
                 .spawn(move || {
                     #[derive(Clone, Copy, Debug)]

From 453be3de60908e8d2f2314b5ae3d81e07f86086e Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 4 Sep 2022 13:49:06 +0900
Subject: [PATCH 1971/3199] save

---
 scheduler/src/lib.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 72bd7372f91e75..a726d67221daf0 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1312,7 +1312,7 @@ impl ScheduleStage {
             .unwrap_or(format!("{}", 4))
             .parse::<usize>()
             .unwrap();
-        for thx in 0..indexer_count {
+        let indexer_handles = (0..indexer_count).iter().map({
             let task_receiver = task_receiver.clone();
             let h = std::thread::Builder::new()
                 .name(format!("sol-indexer{:02}", thx))
@@ -1330,7 +1330,7 @@ impl ScheduleStage {
                     assert_eq!(task_receiver.len(), 0);
                 })
                 .unwrap();
-        }
+        }).collect::<Vec<_>();
         let mut start = std::time::Instant::now();
 
         let (mut from_disconnected, mut from_exec_disconnected) = (false, false);

From f5d3953cc8e61b63d202255c4ba683ec07c01171 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 4 Sep 2022 13:49:16 +0900
Subject: [PATCH 1972/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index a726d67221daf0..f3aafde4351aea 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1330,7 +1330,7 @@ impl ScheduleStage {
                     assert_eq!(task_receiver.len(), 0);
                 })
                 .unwrap();
-        }).collect::<Vec<_>();
+        }).collect::<Vec<_>>();
         let mut start = std::time::Instant::now();
 
         let (mut from_disconnected, mut from_exec_disconnected) = (false, false);

From fe76401c51d0b9993e83e426757020f4860fb4dd Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 4 Sep 2022 13:49:28 +0900
Subject: [PATCH 1973/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index f3aafde4351aea..00a57597d5ed89 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1312,7 +1312,7 @@ impl ScheduleStage {
             .unwrap_or(format!("{}", 4))
             .parse::<usize>()
             .unwrap();
-        let indexer_handles = (0..indexer_count).iter().map({
+        let indexer_handles = (0..indexer_count).map({
             let task_receiver = task_receiver.clone();
             let h = std::thread::Builder::new()
                 .name(format!("sol-indexer{:02}", thx))

From bc6c62033ffc103925250d631e421394eabc7b04 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 4 Sep 2022 13:49:42 +0900
Subject: [PATCH 1974/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 00a57597d5ed89..4c6e8c942456c7 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1329,7 +1329,7 @@ impl ScheduleStage {
                     }
                     assert_eq!(task_receiver.len(), 0);
                 })
-                .unwrap();
+                .unwrap()
         }).collect::<Vec<_>>();
         let mut start = std::time::Instant::now();
 

From 34a54ea5bca069e724cb2895ea0829201c4c775a Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 4 Sep 2022 13:50:14 +0900
Subject: [PATCH 1975/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 4c6e8c942456c7..42db623329d9c5 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1312,7 +1312,7 @@ impl ScheduleStage {
             .unwrap_or(format!("{}", 4))
             .parse::<usize>()
             .unwrap();
-        let indexer_handles = (0..indexer_count).map({
+        let indexer_handles = (0..indexer_count).map(|thx| {
             let task_receiver = task_receiver.clone();
             let h = std::thread::Builder::new()
                 .name(format!("sol-indexer{:02}", thx))

From cf57db90253a362382f281ffea902a6d928fb953 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 4 Sep 2022 13:50:34 +0900
Subject: [PATCH 1976/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 42db623329d9c5..1c46a1050c542e 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1314,7 +1314,7 @@ impl ScheduleStage {
             .unwrap();
         let indexer_handles = (0..indexer_count).map(|thx| {
             let task_receiver = task_receiver.clone();
-            let h = std::thread::Builder::new()
+            std::thread::Builder::new()
                 .name(format!("sol-indexer{:02}", thx))
                 .spawn(move || {
                     while let Ok((task, ll)) = task_receiver.recv() {

From 296457526defede3b71474b0c3623c9201f37c89 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 4 Sep 2022 13:51:32 +0900
Subject: [PATCH 1977/3199] save

---
 scheduler/src/lib.rs | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 1c46a1050c542e..3580a0a112bc78 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1444,6 +1444,10 @@ impl ScheduleStage {
         if let Some(h) = maybe_reaper_thread_handle {
             h.join().unwrap().unwrap();
         }
+        for indexer_handle in indexer_handles {
+            indexer_handle.join().unwrap().unwrap();
+        }
+
         info!("run finished...");
         info!("schedule_once (from: {}, to: {}, runnnable: {}, contended: {}, (immediate+provisional)/max: ({}+{})/{}) active from contended: {} stuck: {}!", from_prev.len(), to_execute_substage.len(), runnable_queue.task_count(), contended_count, executing_queue_count, provisioning_tracker_count, max_executing_queue_count, address_book.uncontended_task_ids.len(), address_book.stuck_tasks.len());
     }

From 2fbca820cc2602ee5969bad724c562f1b1115be8 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 4 Sep 2022 13:51:49 +0900
Subject: [PATCH 1978/3199] save

---
 scheduler/src/lib.rs | 1 +
 1 file changed, 1 insertion(+)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 3580a0a112bc78..393cb7951fbc71 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1328,6 +1328,7 @@ impl ScheduleStage {
                         }
                     }
                     assert_eq!(task_receiver.len(), 0);
+                    Ok(())
                 })
                 .unwrap()
         }).collect::<Vec<_>>();

From 889b58d6aa33fab1b59e9459f5643e15d3f2744e Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 4 Sep 2022 13:52:09 +0900
Subject: [PATCH 1979/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 393cb7951fbc71..104b9da92f2bb4 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1328,7 +1328,7 @@ impl ScheduleStage {
                         }
                     }
                     assert_eq!(task_receiver.len(), 0);
-                    Ok(())
+                    Ok::<(), ()>(())
                 })
                 .unwrap()
         }).collect::<Vec<_>>();

From 67dea0a92864e6970c68afad577a084f59f0f352 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 4 Sep 2022 13:52:31 +0900
Subject: [PATCH 1980/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 104b9da92f2bb4..11489cdc4b995f 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1300,7 +1300,7 @@ impl ScheduleStage {
                         //TaskInQueue::get_mut(&mut a.task).unwrap();
                     }
                     assert_eq!(ee_receiver.len(), 0);
-                    Ok(())
+                    Ok::<(), ()>(())
                 })
                 .unwrap();
 

From 4fbd24e37350de9df4a7850cdd41de83e7e14457 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 4 Sep 2022 13:52:45 +0900
Subject: [PATCH 1981/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 11489cdc4b995f..d1ee297b791667 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1284,7 +1284,7 @@ impl ScheduleStage {
         let (to_next_stage, maybe_reaper_thread_handle) = if let Some(to_next_stage) = maybe_to_next_stage {
             (to_next_stage, None)
         } else {
-            let h: std::thread::JoinHandle::<Result<(), ()>> = std::thread::Builder::new()
+            let h = std::thread::Builder::new()
                 .name("sol-reaper".to_string())
                 .spawn(move || {
                     #[derive(Clone, Copy, Debug)]

From 2f69a881dc2c5b8326ca2c24c5ab8a25fc61c224 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 4 Sep 2022 14:14:22 +0900
Subject: [PATCH 1982/3199] save

---
 Cargo.lock                              |  1 +
 scheduler/src/lib.rs                    |  4 +++-
 transaction-scheduler-bench/Cargo.toml  |  1 +
 transaction-scheduler-bench/src/main.rs | 30 +++++++++++++++++--------
 4 files changed, 26 insertions(+), 10 deletions(-)

diff --git a/Cargo.lock b/Cargo.lock
index 90b0cadb881347..77d1e2af47e007 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -6643,6 +6643,7 @@ dependencies = [
  "solana-logger 1.12.0",
  "solana-measure",
  "solana-perf",
+ "solana-program-runtime",
  "solana-runtime",
  "solana-scheduler",
  "solana-sdk 1.12.0",
diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index d1ee297b791667..b823e652500a33 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1442,6 +1442,9 @@ impl ScheduleStage {
         }
         drop(to_next_stage);
         drop(ee_sender);
+        drop(task_sender);
+        drop(task_receiver);
+        info!("run finished...");
         if let Some(h) = maybe_reaper_thread_handle {
             h.join().unwrap().unwrap();
         }
@@ -1449,7 +1452,6 @@ impl ScheduleStage {
             indexer_handle.join().unwrap().unwrap();
         }
 
-        info!("run finished...");
         info!("schedule_once (from: {}, to: {}, runnnable: {}, contended: {}, (immediate+provisional)/max: ({}+{})/{}) active from contended: {} stuck: {}!", from_prev.len(), to_execute_substage.len(), runnable_queue.task_count(), contended_count, executing_queue_count, provisioning_tracker_count, max_executing_queue_count, address_book.uncontended_task_ids.len(), address_book.stuck_tasks.len());
     }
 
diff --git a/transaction-scheduler-bench/Cargo.toml b/transaction-scheduler-bench/Cargo.toml
index 1262c6e3f98468..dbb567b35f58da 100644
--- a/transaction-scheduler-bench/Cargo.toml
+++ b/transaction-scheduler-bench/Cargo.toml
@@ -23,5 +23,6 @@ solana-logger = { path = "../logger" }
 solana-measure = { path = "../measure" }
 solana-perf = { path = "../perf" }
 solana-runtime = { path = "../runtime" }
+solana-program-runtime = { path = "../program-runtime" }
 solana-scheduler = { path = "../scheduler" }
 solana-sdk = { path = "../sdk" }
diff --git a/transaction-scheduler-bench/src/main.rs b/transaction-scheduler-bench/src/main.rs
index 4dc5f65c3197ce..5f493bc65f4c25 100644
--- a/transaction-scheduler-bench/src/main.rs
+++ b/transaction-scheduler-bench/src/main.rs
@@ -80,9 +80,9 @@ struct Args {
 type PreprocessedTransaction = (SanitizedTransaction, Vec<solana_scheduler::LockAttempt>);
 type TransactionMessage = PreprocessedTransaction;
 //type CompletedTransactionMessage = solana_scheduler::Multiplexed; // (usize, Box<solana_scheduler::ExecutionEnvironment>); //(usize, TransactionMessage); // thread index and transaction message
-type CompletedTransactionMessage = Box<solana_scheduler::ExecutionEnvironment>;
-type TransactionBatchMessage = Box<solana_scheduler::ExecutionEnvironment>; // Vec<TransactionMessage>;
-type BatchSenderMessage = solana_scheduler::TaskInQueue; // Vec<Vec<PreprocessedTransaction>>;
+type CompletedTransactionMessage = solana_scheduler::UnlockablePayload; //Box<solana_scheduler::ExecutionEnvironment>;
+type TransactionBatchMessage = solana_scheduler::ExecutablePayload; // Box<solana_scheduler::ExecutionEnvironment>; // Vec<TransactionMessage>;
+type BatchSenderMessage = solana_scheduler::SchedulablePayload; // solana_scheduler::TaskInQueue; // Vec<Vec<PreprocessedTransaction>>;
 
 #[derive(Debug, Default)]
 struct TransactionSchedulerBenchMetrics {
@@ -119,7 +119,7 @@ struct PacketSendingConfig {
 fn spawn_unified_scheduler(
         mut address_book: solana_scheduler::AddressBook,
         num_execution_threads: usize,
-        packet_batch_receiver: Receiver<BatchSenderMessage>,
+        packet_batch_receiver: Receiver<solana_scheduler::SchedulablePayload>,
         transaction_batch_senders: Vec<Sender<TransactionBatchMessage>>,
         completed_transaction_receiver: Receiver<CompletedTransactionMessage>,
         bank_forks: Arc<RwLock<BankForks>>,
@@ -351,7 +351,7 @@ fn handle_transaction_batch(
     ));
 
     use solana_runtime::transaction_priority_details::GetTransactionPriorityDetails;
-    let priority_collected = transaction_batch.task.tx.0.get_transaction_priority_details().unwrap().priority;
+    let priority_collected = transaction_batch.0.task.tx.0.get_transaction_priority_details().unwrap().priority;
 
     metrics
         .num_transactions_completed
@@ -360,9 +360,9 @@ fn handle_transaction_batch(
         .priority_collected
         .fetch_add(priority_collected, Ordering::Relaxed);
 
-    transaction_batch.reindex_to_address_book();
+    transaction_batch.0.reindex_to_address_book();
     completed_transaction_sender
-        .send(transaction_batch)
+        .send(solana_scheduler::UnlockablePayload(transaction_batch.0))
         .unwrap();
 }
 
@@ -434,6 +434,18 @@ fn spawn_packet_sender(
     }).unwrap()
 }
 
+pub fn get_transaction_priority_details(tx: &SanitizedTransaction) -> u64 {
+    use solana_program_runtime::compute_budget::ComputeBudget;
+    let mut compute_budget = ComputeBudget::default();
+    compute_budget
+        .process_instructions(
+            tx.message().program_instructions_iter(),
+            true, // use default units per instruction
+        )
+        .map(|d| d.get_priority())
+        .unwrap_or_default()
+}
+
 fn send_packets(
     unique_weight: std::sync::Arc<std::sync::atomic::AtomicU64>,
     producer_count: usize,
@@ -483,12 +495,12 @@ fn send_packets(
 
         for vv in packet_batches {
             for v in vv {
-                let p = solana_scheduler::get_transaction_priority_details(&v.0);
+                let p = get_transaction_priority_details(&v.0);
                 //let p = (p << 32) | (rng.gen::<u64>() & 0x0000_0000_ffff_ffff);
                 let uw = unique_weight.fetch_sub(1, std::sync::atomic::Ordering::SeqCst);
                 let p = (p << 32) | (uw & 0x0000_0000_ffff_ffff);
                 let t = solana_scheduler::Task::new_for_queue(nast, p, v);
-                packet_batch_sender.send(t).unwrap();
+                packet_batch_sender.send(solana_scheduler::SchedulablePayload(t)).unwrap();
             }
         }
         

From ca0de2387e3c3fd4ca79eb6dfbe9040a9dae352c Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 4 Sep 2022 21:34:55 +0900
Subject: [PATCH 1983/3199] save

---
 runtime/src/bank.rs | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 54c33551723051..3d4c9aab6419e1 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1245,7 +1245,7 @@ impl Default for Scheduler {
             let mut runnable_queue = solana_scheduler::TaskQueue::default();
 
             solana_scheduler::ScheduleStage::run(
-                100,
+                123,
                 &mut runnable_queue,
                 &mut address_book,
                 &transaction_receiver,
@@ -1269,6 +1269,9 @@ impl Default for Scheduler {
     }
 }
 
+impl Drop for Scheduler {
+}
+
 /// Manager for the state of all accounts and programs after processing its entries.
 /// AbiExample is needed even without Serialize/Deserialize; actual (de-)serialization
 /// are implemented elsewhere for versioning

From 7a53a1e48ea5952e0dd1df5e87b13bba6e70c3ed Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 4 Sep 2022 21:37:57 +0900
Subject: [PATCH 1984/3199] save

---
 runtime/src/bank.rs | 22 ++++++++++++++++------
 1 file changed, 16 insertions(+), 6 deletions(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 3d4c9aab6419e1..0063f0097a1487 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1269,7 +1269,23 @@ impl Default for Scheduler {
     }
 }
 
+impl Scheduler {
+    fn gracefully_stop(&mut self) -> Result<()> {
+        let mut transaction_sender = self.transaction_sender.take().unwrap();
+        drop(transaction_sender);
+        let h = self.executing_thread_handle.take().unwrap();
+        h.join().unwrap()?;
+        let h = self.scheduler_thread_handle.take().unwrap();
+        h.join().unwrap()?;
+
+        Ok(())
+    }
+}
+
 impl Drop for Scheduler {
+    fn drop(&mut self) {
+        self.gracefully_stop().unwrap();
+    }
 }
 
 /// Manager for the state of all accounts and programs after processing its entries.
@@ -7940,12 +7956,6 @@ impl Bank {
 
     pub fn wait_for_scheduler(&self) -> Result<()> {
         let mut scheduler = self.scheduler.write().unwrap();
-        let mut transaction_sender = scheduler.transaction_sender.take().unwrap();
-        drop(transaction_sender);
-        let h = scheduler.executing_thread_handle.take().unwrap();
-        h.join().unwrap()?;
-        let h = scheduler.scheduler_thread_handle.take().unwrap();
-        h.join().unwrap()?;
 
         Ok(())
     }

From dbc0497a5126d12ffd13c013e34e2d625a4ea7dd Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 4 Sep 2022 21:38:39 +0900
Subject: [PATCH 1985/3199] save

---
 runtime/src/bank.rs | 1 +
 1 file changed, 1 insertion(+)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 0063f0097a1487..76ebe98da571c9 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -7956,6 +7956,7 @@ impl Bank {
 
     pub fn wait_for_scheduler(&self) -> Result<()> {
         let mut scheduler = self.scheduler.write().unwrap();
+        scheduler.gracefully_stop()?;
 
         Ok(())
     }

From 580a6a83033e07d8afaaf4539665b870f93dba3a Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 4 Sep 2022 21:58:01 +0900
Subject: [PATCH 1986/3199] save

---
 runtime/src/bank.rs | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 76ebe98da571c9..79418bbc674551 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1202,6 +1202,7 @@ struct Scheduler {
     executing_thread_handle: Option<std::thread::JoinHandle<Result<()>>>,
     transaction_sender: Option<crossbeam_channel::Sender<solana_scheduler::SchedulablePayload>>,
     preloader: Arc<solana_scheduler::Preloader>,
+    graceful_stop_initiated: bool,
 }
 
 impl Scheduler {
@@ -1265,12 +1266,19 @@ impl Default for Scheduler {
             executing_thread_handle: Some(executing_thread_handle),
             transaction_sender: Some(transaction_sender),
             preloader,
+            graceful_stop_initiated: Default::default(),
         }
     }
 }
 
 impl Scheduler {
     fn gracefully_stop(&mut self) -> Result<()> {
+        if self.graceful_stop_initiated {
+            return Ok(());
+        }
+        self.graceful_stop_initiated = true;
+
+        info!("Scheduler::gracefully_stop()..");
         let mut transaction_sender = self.transaction_sender.take().unwrap();
         drop(transaction_sender);
         let h = self.executing_thread_handle.take().unwrap();
@@ -1284,7 +1292,9 @@ impl Scheduler {
 
 impl Drop for Scheduler {
     fn drop(&mut self) {
+        info!("Scheduler::drop(): begin..");
         self.gracefully_stop().unwrap();
+        info!("Scheduler::drop(): end...");
     }
 }
 

From 15c3ab4241b6f577ea23c6e769a25fd1b6feabfb Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 4 Sep 2022 22:02:46 +0900
Subject: [PATCH 1987/3199] save

---
 runtime/src/bank.rs | 20 ++++++++++----------
 1 file changed, 10 insertions(+), 10 deletions(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 79418bbc674551..f1d1d16dbf0c38 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -4612,16 +4612,6 @@ impl Bank {
         }
     }
 
-    pub fn schedule_transaction_batch_to_be_commited(
-        &self,
-        batch: &TransactionBatch,
-    ) {
-        let scheduler = self.scheduler.read().unwrap();
-        for st in batch.sanitized_transactions() {
-            scheduler.schedule(st);
-        }
-    }
-
     #[allow(clippy::type_complexity)]
     pub fn load_and_execute_transactions(
         &self,
@@ -6317,6 +6307,16 @@ impl Bank {
         self.cluster_type.unwrap()
     }
 
+    pub fn schedule_transaction_to_commit(
+        &self,
+        batch: &TransactionBatch,
+    ) {
+        let scheduler = self.scheduler.read().unwrap();
+        for st in batch.sanitized_transactions() {
+            scheduler.schedule(st);
+        }
+    }
+
     /// Process a batch of transactions.
     #[must_use]
     pub fn load_execute_and_commit_transactions(

From 5ba00ea88ceb7eac075d2fd847afc48288d5f701 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 4 Sep 2022 22:04:17 +0900
Subject: [PATCH 1988/3199] save

---
 runtime/src/bank.rs | 1 +
 1 file changed, 1 insertion(+)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index f1d1d16dbf0c38..a0b4de1182ec22 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -3530,6 +3530,7 @@ impl Bank {
         // committed before this write lock can be obtained here.
         let mut hash = self.hash.write().unwrap();
         if *hash == Hash::default() {
+            assert!(self.scheduler.graceful_stop_initiated);
             // finish up any deferred changes to account state
             self.collect_rent_eagerly(false);
             self.collect_fees();

From 68bad91401f0467594d42175ebbd8d03d274e9b6 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 4 Sep 2022 22:04:37 +0900
Subject: [PATCH 1989/3199] save

---
 runtime/src/bank.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index a0b4de1182ec22..63a894d2535f40 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -3530,7 +3530,7 @@ impl Bank {
         // committed before this write lock can be obtained here.
         let mut hash = self.hash.write().unwrap();
         if *hash == Hash::default() {
-            assert!(self.scheduler.graceful_stop_initiated);
+            assert!(self.scheduler.read().unwrap().graceful_stop_initiated);
             // finish up any deferred changes to account state
             self.collect_rent_eagerly(false);
             self.collect_fees();

From 5c7a3e41b5cd4df9644ec252a82ee7bf3ea62fa7 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 4 Sep 2022 22:30:29 +0900
Subject: [PATCH 1990/3199] save

---
 ledger/src/blockstore_processor.rs | 1 +
 1 file changed, 1 insertion(+)

diff --git a/ledger/src/blockstore_processor.rs b/ledger/src/blockstore_processor.rs
index 29660d2fae6601..72d38d0dc49284 100644
--- a/ledger/src/blockstore_processor.rs
+++ b/ledger/src/blockstore_processor.rs
@@ -1317,6 +1317,7 @@ fn process_bank_0(
         &mut ExecuteTimings::default(),
     )
     .expect("Failed to process bank 0 from ledger. Did you forget to provide a snapshot?");
+    bank0.wait_for_scheduler().unwrap();
     bank0.freeze();
     if blockstore.is_primary_access() {
         blockstore.insert_bank_hash(bank0.slot(), bank0.hash(), false);

From 2eb71057d19497abee555e095680cafcc6a418ac Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 4 Sep 2022 22:41:49 +0900
Subject: [PATCH 1991/3199] save

---
 ledger/src/blockstore_processor.rs | 1 +
 1 file changed, 1 insertion(+)

diff --git a/ledger/src/blockstore_processor.rs b/ledger/src/blockstore_processor.rs
index 72d38d0dc49284..5c9d03e0fd07f7 100644
--- a/ledger/src/blockstore_processor.rs
+++ b/ledger/src/blockstore_processor.rs
@@ -1666,6 +1666,7 @@ fn process_single_slot(
         err
     })?;
 
+    bank.wait_for_scheduler().unwrap();
     bank.freeze(); // all banks handled by this routine are created from complete slots
     if blockstore.is_primary_access() {
         blockstore.insert_bank_hash(bank.slot(), bank.hash(), false);

From 97c2bb5d4e65aea4b183c02c8089ea6edfbcfe26 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 4 Sep 2022 23:03:40 +0900
Subject: [PATCH 1992/3199] save

---
 runtime/src/bank.rs | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 63a894d2535f40..43f4d6836f27d1 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1206,7 +1206,7 @@ struct Scheduler {
 }
 
 impl Scheduler {
-    fn schedule(&self, sanitized_tx: &SanitizedTransaction) {
+    fn schedule(&self, sanitized_tx: &SanitizedTransaction, index: usize) {
         #[derive(Clone, Copy, Debug)]
         struct NotAtTopOfScheduleThread;
         unsafe impl solana_scheduler::NotAtScheduleThread for NotAtTopOfScheduleThread {}
@@ -1223,7 +1223,7 @@ impl Scheduler {
             .map(|address| solana_scheduler::LockAttempt::new(self.preloader.load(**address), solana_scheduler::RequestedUsage::Readonly));
         let locks = writable_lock_iter.chain(readonly_lock_iter).collect::<Vec<_>>();
 
-        let t = solana_scheduler::Task::new_for_queue(nast, 0, (sanitized_tx.clone(), locks));
+        let t = solana_scheduler::Task::new_for_queue(nast, usize::max_value() - index, (sanitized_tx.clone(), locks));
         self.transaction_sender.as_ref().unwrap().send(solana_scheduler::SchedulablePayload(t)).unwrap();
     }
 }
@@ -6308,13 +6308,14 @@ impl Bank {
         self.cluster_type.unwrap()
     }
 
-    pub fn schedule_transaction_to_commit(
+    pub fn schedule_and_commit_transactions(
         &self,
         batch: &TransactionBatch,
+        transaction_indexes: &Vec<usize>,
     ) {
         let scheduler = self.scheduler.read().unwrap();
-        for st in batch.sanitized_transactions() {
-            scheduler.schedule(st);
+        for (st, i) in batch.sanitized_transactions().zip(transaction_indexes.iter()) {
+            scheduler.schedule(st, i);
         }
     }
 

From f460050a9102d3dd3e277961e40524d95a80069a Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 4 Sep 2022 23:04:19 +0900
Subject: [PATCH 1993/3199] save

---
 runtime/src/bank.rs | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 43f4d6836f27d1..bed687a54bd259 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1223,7 +1223,8 @@ impl Scheduler {
             .map(|address| solana_scheduler::LockAttempt::new(self.preloader.load(**address), solana_scheduler::RequestedUsage::Readonly));
         let locks = writable_lock_iter.chain(readonly_lock_iter).collect::<Vec<_>>();
 
-        let t = solana_scheduler::Task::new_for_queue(nast, usize::max_value() - index, (sanitized_tx.clone(), locks));
+        let uw = usize::max_value() - index;
+        let t = solana_scheduler::Task::new_for_queue(nast, uw as u64, (sanitized_tx.clone(), locks));
         self.transaction_sender.as_ref().unwrap().send(solana_scheduler::SchedulablePayload(t)).unwrap();
     }
 }

From 3989cd6fe2c1478992b170333a0eefaf59e4c4da Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 4 Sep 2022 23:04:41 +0900
Subject: [PATCH 1994/3199] save

---
 runtime/src/bank.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index bed687a54bd259..d8327e939fa316 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -6315,7 +6315,7 @@ impl Bank {
         transaction_indexes: &Vec<usize>,
     ) {
         let scheduler = self.scheduler.read().unwrap();
-        for (st, i) in batch.sanitized_transactions().zip(transaction_indexes.iter()) {
+        for (st, i) in batch.sanitized_transactions().iter().zip(transaction_indexes.iter()) {
             scheduler.schedule(st, i);
         }
     }

From 640ff36cabcd8a8d44b47efc74e2ecef65270df8 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 4 Sep 2022 23:05:01 +0900
Subject: [PATCH 1995/3199] save

---
 runtime/src/bank.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index d8327e939fa316..5023b95f162572 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -6315,7 +6315,7 @@ impl Bank {
         transaction_indexes: &Vec<usize>,
     ) {
         let scheduler = self.scheduler.read().unwrap();
-        for (st, i) in batch.sanitized_transactions().iter().zip(transaction_indexes.iter()) {
+        for (st, &i) in batch.sanitized_transactions().iter().zip(transaction_indexes.iter()) {
             scheduler.schedule(st, i);
         }
     }

From cdbc93cd30dd183d38366ab208ede391019e13cf Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 4 Sep 2022 23:06:33 +0900
Subject: [PATCH 1996/3199] save

---
 ledger/src/blockstore_processor.rs | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/ledger/src/blockstore_processor.rs b/ledger/src/blockstore_processor.rs
index 5c9d03e0fd07f7..8be530f0a35a8e 100644
--- a/ledger/src/blockstore_processor.rs
+++ b/ledger/src/blockstore_processor.rs
@@ -190,6 +190,10 @@ fn execute_batch(
 
     let pre_process_units: u64 = aggregate_total_execution_units(timings);
 
+    batch.bank().schedule_and_commit_transactions(
+        batch,
+        transaction_indexes,
+    );
     let (tx_results, balances) = batch.bank().load_execute_and_commit_transactions(
         batch,
         MAX_PROCESSING_AGE,

From b09c3647ac2470681c4962d3414f5f0de898eb03 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 5 Sep 2022 09:20:20 +0900
Subject: [PATCH 1997/3199] save

---
 scheduler/src/lib.rs | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index b823e652500a33..b4bd97ba256d6b 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1279,6 +1279,7 @@ impl ScheduleStage {
         let mut sequence_time = 0;
         let mut queue_clock = 0;
         let mut execute_clock = 0;
+        let mut completed_count = 0_usize;
         let (ee_sender, ee_receiver) = crossbeam_channel::unbounded::<PersistablePlayload>();
 
         let (to_next_stage, maybe_reaper_thread_handle) = if let Some(to_next_stage) = maybe_to_next_stage {
@@ -1349,6 +1350,7 @@ impl ScheduleStage {
                    }
                    let mut processed_execution_environment = maybe_from_exec.unwrap().0;
                     executing_queue_count = executing_queue_count.checked_sub(1).unwrap();
+                    completed_count = completed_count.checked_add(1).unwrap();
                     Self::commit_completed_execution(ast, &mut processed_execution_environment, address_book, &mut execute_clock, &mut provisioning_tracker_count);
                     to_next_stage.send(PersistablePlayload(processed_execution_environment)).unwrap();
                }
@@ -1376,10 +1378,10 @@ impl ScheduleStage {
                 while (executing_queue_count + provisioning_tracker_count)
                     < max_executing_queue_count
                 {
-                    trace!("schedule_once (from: {}, to: {}, runnnable: {}, contended: {}, (immediate+provisional)/max: ({}+{})/{}) active from contended: {} stuck: {}!", from_prev.len(), to_execute_substage.len(), runnable_queue.task_count(), contended_count, executing_queue_count, provisioning_tracker_count, max_executing_queue_count, address_book.uncontended_task_ids.len(), address_book.stuck_tasks.len());
+                    trace!("schedule_once (from: {}, to: {}, runnnable: {}, contended: {}, (immediate+provisional)/max: ({}+{})/{}) active from contended: {} stuck: {} completed: {}!", from_prev.len(), to_execute_substage.len(), runnable_queue.task_count(), contended_count, executing_queue_count, provisioning_tracker_count, max_executing_queue_count, address_book.uncontended_task_ids.len(), address_book.stuck_tasks.len(), completed_count);
                     if start.elapsed() > std::time::Duration::from_millis(1000) {
                         start = std::time::Instant::now();
-                        info!("schedule_once (from: {}, to: {}, runnnable: {}, contended: {}, (immediate+provisional)/max: ({}+{})/{}) active from contended: {} stuck: {}!", from_prev.len(), to_execute_substage.len(), runnable_queue.task_count(), contended_count, executing_queue_count, provisioning_tracker_count, max_executing_queue_count, address_book.uncontended_task_ids.len(), address_book.stuck_tasks.len());
+                        info!("schedule_once (from: {}, to: {}, runnnable: {}, contended: {}, (immediate+provisional)/max: ({}+{})/{}) active from contended: {} stuck: {} completed: {}!", from_prev.len(), to_execute_substage.len(), runnable_queue.task_count(), contended_count, executing_queue_count, provisioning_tracker_count, max_executing_queue_count, address_book.uncontended_task_ids.len(), address_book.stuck_tasks.len(), completed_count);
                     }
                     let prefer_immediate = provisioning_tracker_count / 4 > executing_queue_count;
                     if let Some(ee) = Self::schedule_next_execution(
@@ -1422,6 +1424,7 @@ impl ScheduleStage {
                         from_exec_len = from_exec_len.checked_sub(1).unwrap();
                         empty_from_exec = from_exec_len == 0;
                         executing_queue_count = executing_queue_count.checked_sub(1).unwrap();
+                        completed_count = completed_count.checked_add(1).unwrap();
                         Self::commit_completed_execution(
                             ast,
                             &mut processed_execution_environment,
@@ -1452,7 +1455,7 @@ impl ScheduleStage {
             indexer_handle.join().unwrap().unwrap();
         }
 
-        info!("schedule_once (from: {}, to: {}, runnnable: {}, contended: {}, (immediate+provisional)/max: ({}+{})/{}) active from contended: {} stuck: {}!", from_prev.len(), to_execute_substage.len(), runnable_queue.task_count(), contended_count, executing_queue_count, provisioning_tracker_count, max_executing_queue_count, address_book.uncontended_task_ids.len(), address_book.stuck_tasks.len());
+        info!("schedule_once (from: {}, to: {}, runnnable: {}, contended: {}, (immediate+provisional)/max: ({}+{})/{}) active from contended: {} stuck: {} completed: {}!", from_prev.len(), to_execute_substage.len(), runnable_queue.task_count(), contended_count, executing_queue_count, provisioning_tracker_count, max_executing_queue_count, address_book.uncontended_task_ids.len(), address_book.stuck_tasks.len(), completed_count);
     }
 
     pub fn run(

From 39c8cc23aaccea3f88340c062a97e1c8e0882e77 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 5 Sep 2022 09:21:16 +0900
Subject: [PATCH 1998/3199] save

---
 runtime/src/bank.rs  | 2 +-
 scheduler/src/lib.rs | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 5023b95f162572..ab62b48af1a46e 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1236,7 +1236,7 @@ impl Default for Scheduler {
         let (transaction_sender, transaction_receiver) = crossbeam_channel::unbounded();
         let (scheduled_ee_sender, scheduled_ee_receiver) = crossbeam_channel::unbounded();
         let (completed_ee_sender, completed_ee_receiver) = crossbeam_channel::unbounded();
-        let executing_thread_handle = std::thread::Builder::new().name(format!("sol-exec-{}", 0)).spawn(move || {
+        let executing_thread_handle = std::thread::Builder::new().name(format!("solExec{:02}", 0)).spawn(move || {
             while let Ok(solana_scheduler::ExecutablePayload(ee)) = scheduled_ee_receiver.recv() {
                 completed_ee_sender.send(solana_scheduler::UnlockablePayload(ee)).unwrap();
             }
diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index b4bd97ba256d6b..67c6fc9ce8ddc5 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1286,7 +1286,7 @@ impl ScheduleStage {
             (to_next_stage, None)
         } else {
             let h = std::thread::Builder::new()
-                .name("sol-reaper".to_string())
+                .name("solReaper".to_string())
                 .spawn(move || {
                     #[derive(Clone, Copy, Debug)]
                     struct NotAtTopOfScheduleThread;
@@ -1316,7 +1316,7 @@ impl ScheduleStage {
         let indexer_handles = (0..indexer_count).map(|thx| {
             let task_receiver = task_receiver.clone();
             std::thread::Builder::new()
-                .name(format!("sol-indexer{:02}", thx))
+                .name(format!("solIndexer{:02}", thx))
                 .spawn(move || {
                     while let Ok((task, ll)) = task_receiver.recv() {
                         for lock_attempt in ll {

From b1f4a9d989dc8e93c88cd70f3a13a2518ae9cfa0 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 5 Sep 2022 11:20:27 +0900
Subject: [PATCH 1999/3199] save

---
 ledger/src/blockstore_processor.rs | 1 +
 runtime/src/bank.rs                | 3 ++-
 scheduler/src/lib.rs               | 4 ++--
 3 files changed, 5 insertions(+), 3 deletions(-)

diff --git a/ledger/src/blockstore_processor.rs b/ledger/src/blockstore_processor.rs
index 8be530f0a35a8e..66c29d18a7e68c 100644
--- a/ledger/src/blockstore_processor.rs
+++ b/ledger/src/blockstore_processor.rs
@@ -194,6 +194,7 @@ fn execute_batch(
         batch,
         transaction_indexes,
     );
+    // bank().handle_unexpected_errors() -> Option<Vec<_>>
     let (tx_results, balances) = batch.bank().load_execute_and_commit_transactions(
         batch,
         MAX_PROCESSING_AGE,
diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index ab62b48af1a46e..247d5033e7e88f 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1275,11 +1275,12 @@ impl Default for Scheduler {
 impl Scheduler {
     fn gracefully_stop(&mut self) -> Result<()> {
         if self.graceful_stop_initiated {
+            info!("Scheduler::gracefully_stop(): (skipped..?)");
             return Ok(());
         }
         self.graceful_stop_initiated = true;
 
-        info!("Scheduler::gracefully_stop()..");
+        info!("Scheduler::gracefully_stop(): waiting..");
         let mut transaction_sender = self.transaction_sender.take().unwrap();
         drop(transaction_sender);
         let h = self.executing_thread_handle.take().unwrap();
diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 67c6fc9ce8ddc5..0ec1c9c4bd7688 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1381,7 +1381,7 @@ impl ScheduleStage {
                     trace!("schedule_once (from: {}, to: {}, runnnable: {}, contended: {}, (immediate+provisional)/max: ({}+{})/{}) active from contended: {} stuck: {} completed: {}!", from_prev.len(), to_execute_substage.len(), runnable_queue.task_count(), contended_count, executing_queue_count, provisioning_tracker_count, max_executing_queue_count, address_book.uncontended_task_ids.len(), address_book.stuck_tasks.len(), completed_count);
                     if start.elapsed() > std::time::Duration::from_millis(1000) {
                         start = std::time::Instant::now();
-                        info!("schedule_once (from: {}, to: {}, runnnable: {}, contended: {}, (immediate+provisional)/max: ({}+{})/{}) active from contended: {} stuck: {} completed: {}!", from_prev.len(), to_execute_substage.len(), runnable_queue.task_count(), contended_count, executing_queue_count, provisioning_tracker_count, max_executing_queue_count, address_book.uncontended_task_ids.len(), address_book.stuck_tasks.len(), completed_count);
+                        info!("schedule_once:interval (from: {}, to: {}, runnnable: {}, contended: {}, (immediate+provisional)/max: ({}+{})/{}) active from contended: {} stuck: {} completed: {}!", from_prev.len(), to_execute_substage.len(), runnable_queue.task_count(), contended_count, executing_queue_count, provisioning_tracker_count, max_executing_queue_count, address_book.uncontended_task_ids.len(), address_book.stuck_tasks.len(), completed_count);
                     }
                     let prefer_immediate = provisioning_tracker_count / 4 > executing_queue_count;
                     if let Some(ee) = Self::schedule_next_execution(
@@ -1455,7 +1455,7 @@ impl ScheduleStage {
             indexer_handle.join().unwrap().unwrap();
         }
 
-        info!("schedule_once (from: {}, to: {}, runnnable: {}, contended: {}, (immediate+provisional)/max: ({}+{})/{}) active from contended: {} stuck: {} completed: {}!", from_prev.len(), to_execute_substage.len(), runnable_queue.task_count(), contended_count, executing_queue_count, provisioning_tracker_count, max_executing_queue_count, address_book.uncontended_task_ids.len(), address_book.stuck_tasks.len(), completed_count);
+        info!("schedule_once:final (from: {}, to: {}, runnnable: {}, contended: {}, (immediate+provisional)/max: ({}+{})/{}) active from contended: {} stuck: {} completed: {}!", from_prev.len(), to_execute_substage.len(), runnable_queue.task_count(), contended_count, executing_queue_count, provisioning_tracker_count, max_executing_queue_count, address_book.uncontended_task_ids.len(), address_book.stuck_tasks.len(), completed_count);
     }
 
     pub fn run(

From 6d8fce6e3d2390df7c57a8aeb1ea90cd688dd9ea Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 5 Sep 2022 11:21:06 +0900
Subject: [PATCH 2000/3199] save

---
 runtime/src/bank.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 247d5033e7e88f..e02b696436c1d8 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1247,7 +1247,7 @@ impl Default for Scheduler {
             let mut runnable_queue = solana_scheduler::TaskQueue::default();
 
             solana_scheduler::ScheduleStage::run(
-                123,
+                1,
                 &mut runnable_queue,
                 &mut address_book,
                 &transaction_receiver,

From 00dbe27e5185aae950cf4ac28af85b88a4de6e2a Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 5 Sep 2022 11:46:35 +0900
Subject: [PATCH 2001/3199] save

---
 scheduler/src/lib.rs | 39 +++++++++++++++++++++++++--------------
 1 file changed, 25 insertions(+), 14 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 0ec1c9c4bd7688..d1d964f6d4c407 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -808,6 +808,13 @@ pub fn get_transaction_priority_details(tx: &SanitizedTransaction) -> u64 {
 
 pub struct ScheduleStage {}
 
+#[derive(PartialEq, Eq)]
+enum TaskSource {
+    Runnable,
+    Contended,
+    Stuck,
+}
+
 impl ScheduleStage {
     fn push_to_runnable_queue(task: TaskInQueue, runnable_queue: &mut TaskQueue) {
         runnable_queue.add_to_schedule(task.unique_weight, task);
@@ -825,7 +832,7 @@ impl ScheduleStage {
         runnable_queue: &'a mut TaskQueue,
         address_book: &mut AddressBook,
         contended_count: &usize,
-    ) -> Option<(bool, TaskInQueue)> {
+    ) -> Option<(TaskSource, TaskInQueue)> {
         match (
             runnable_queue.heaviest_entry_to_execute(),
             Self::get_heaviest_from_contended(address_book),
@@ -833,12 +840,12 @@ impl ScheduleStage {
             (Some(heaviest_runnable_entry), None) => {
                 trace!("select: runnable only");
                 let t = heaviest_runnable_entry.remove();
-                Some((true, t))
+                Some((TaskSource::Runnable, t))
             }
             (None, Some(weight_from_contended)) => {
                 trace!("select: contended only");
                 let t = weight_from_contended.remove();
-                Some((false, t))
+                Some((TaskSource::Contended, t))
             }
             (Some(heaviest_runnable_entry), Some(weight_from_contended)) => {
                 let weight_from_runnable = heaviest_runnable_entry.key();
@@ -847,11 +854,11 @@ impl ScheduleStage {
                 if weight_from_runnable > uw {
                     trace!("select: runnable > contended");
                     let t = heaviest_runnable_entry.remove();
-                    Some((true, t))
+                    Some((TaskSource::Runnable, t))
                 } else if uw > weight_from_runnable {
                     trace!("select: contended > runnnable");
                     let t = weight_from_contended.remove();
-                    Some((false, t))
+                    Some((TaskSource::Contended, t))
                 } else {
                     unreachable!(
                         "identical unique weights shouldn't exist in both runnable and contended"
@@ -868,9 +875,10 @@ impl ScheduleStage {
                     assert_eq!(task.stuck_task_id(), stuck_task_id);
 
                     if task.currently_contended() {
-                        Some((false, task))
+                        Some((TaskSource::Stuck, task))
                     } else {
-                        // is it expected for uncontended taks is in the stuck queue?
+                        // is it expected for uncontended tasks is in the stuck queue, to begin
+                        // with??
                         None
                     }
                 } else {
@@ -905,10 +913,11 @@ impl ScheduleStage {
 
         trace!("pop begin");
         loop {
-            if let Some((from_runnable, mut next_task)) =
+            if let Some((task_source, mut next_task)) =
                 Self::select_next_task(runnable_queue, address_book, contended_count)
             {
                 trace!("pop loop iteration");
+                let from_runnable = task_source == TaskSource::Runnable;
                 if from_runnable {
                     next_task.record_queue_time(*sequence_clock, *queue_clock);
                     *queue_clock = queue_clock.checked_add(1).unwrap();
@@ -937,7 +946,6 @@ impl ScheduleStage {
                         address_book,
                         &unique_weight,
                         &mut next_task.lock_attempts_mut(ast),
-                        from_runnable,
                     );
                     let lock_count = next_task.lock_attempts_mut(ast).len();
                     next_task
@@ -971,13 +979,17 @@ impl ScheduleStage {
                         //address_book.uncontended_task_ids.clear();
                     }
 
-                    if from_runnable {
+                    if from_runnable || task_source == TaskSource::Stuck {
                         next_task.update_busiest_page_cu(busiest_page_cu);
                         let a = address_book
                             .stuck_tasks
                             .insert(next_task.stuck_task_id(), Task::clone_in_queue(&next_task));
                         assert!(a.is_none());
-                        continue;
+                        if from_runnable {
+                            continue; // continue to prefer depleting the possibly-non-empty runnable queue
+                        } else {
+                            break; // 
+                        }
                     } else {
                         // todo: remove this task from stuck_tasks before update_busiest_page_cu
                         let removed = address_book
@@ -989,7 +1001,7 @@ impl ScheduleStage {
                             .stuck_tasks
                             .insert(next_task.stuck_task_id(), removed);
                         assert!(a.is_none());
-                        return None;
+                        break;
                     }
                 } else if provisional_count > 0 {
                     assert!(!from_runnable);
@@ -1014,7 +1026,7 @@ impl ScheduleStage {
                         tracker,
                     );
 
-                    return None;
+                    break;
                     continue;
                 }
 
@@ -1076,7 +1088,6 @@ impl ScheduleStage {
         address_book: &mut AddressBook,
         unique_weight: &UniqueWeight,
         lock_attempts: &mut [LockAttempt],
-        from_runnable: bool,
     ) {
         for l in lock_attempts {
             address_book.reset_lock(ast, l, false);

From d00465dac181992548e0221b241a27aad08b9cf4 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 5 Sep 2022 12:01:10 +0900
Subject: [PATCH 2002/3199] save

---
 scheduler/src/lib.rs | 24 +++++++++++++++++-------
 1 file changed, 17 insertions(+), 7 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index d1d964f6d4c407..ef14c9da7d1638 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -980,6 +980,9 @@ impl ScheduleStage {
                     }
 
                     if from_runnable || task_source == TaskSource::Stuck {
+                        // for the case of being struck, we have already removed it from
+                        // stuck_tasks, so pretend to add anew one.
+                        // todo: optimize this needless operation
                         next_task.update_busiest_page_cu(busiest_page_cu);
                         let a = address_book
                             .stuck_tasks
@@ -987,8 +990,15 @@ impl ScheduleStage {
                         assert!(a.is_none());
                         if from_runnable {
                             continue; // continue to prefer depleting the possibly-non-empty runnable queue
+                        } else if task_source == TaskSource::Stuck {
+                            // need to bail out immediately to avoid going to infinite loop of re-processing
+                            // the struck task again.
+                            // todo?: buffer restuck tasks until readd to the stuck tasks until
+                            // some scheduling state tick happens and try 2nd idling stuck task in
+                            // the collection?
+                            break;
                         } else {
-                            break; // 
+                            unreachable!();
                         }
                     } else {
                         // todo: remove this task from stuck_tasks before update_busiest_page_cu
@@ -1346,13 +1356,13 @@ impl ScheduleStage {
         }).collect::<Vec<_>>();
         let mut start = std::time::Instant::now();
 
-        let (mut from_disconnected, mut from_exec_disconnected) = (false, false);
+        let (mut from_disconnected, mut from_exec_disconnected, mut no_more_work) = Default::default();
         loop {
             crossbeam_channel::select! {
                recv(from_exec) -> maybe_from_exec => {
                    if maybe_from_exec.is_err() {
                        assert_eq!(from_exec.len(), 0);
-                       from_exec_disconnected = true;
+                       from_exec_disconnected ||= true;
                        if from_disconnected {
                            break;
                        } else {
@@ -1368,9 +1378,9 @@ impl ScheduleStage {
                recv(from_prev) -> maybe_from => {
                    if maybe_from.is_err() {
                        assert_eq!(from_prev.len(), 0);
-                       from_disconnected = true;
-                       let finished = runnable_queue.task_count() + contended_count + executing_queue_count + provisioning_tracker_count == 0;
-                       if from_exec_disconnected || finished {
+                       from_disconnected ||= true;
+                       no_more_work ||= runnable_queue.task_count() + contended_count + executing_queue_count + provisioning_tracker_count == 0;
+                       if from_exec_disconnected || no_more_work {
                            break;
                        } else {
                            continue;
@@ -1466,7 +1476,7 @@ impl ScheduleStage {
             indexer_handle.join().unwrap().unwrap();
         }
 
-        info!("schedule_once:final (from: {}, to: {}, runnnable: {}, contended: {}, (immediate+provisional)/max: ({}+{})/{}) active from contended: {} stuck: {} completed: {}!", from_prev.len(), to_execute_substage.len(), runnable_queue.task_count(), contended_count, executing_queue_count, provisioning_tracker_count, max_executing_queue_count, address_book.uncontended_task_ids.len(), address_book.stuck_tasks.len(), completed_count);
+        info!("schedule_once:final(from_disconnected: {}, from_exec_disconnected: {}, no_more_work: {}) (from: {}, to: {}, runnnable: {}, contended: {}, (immediate+provisional)/max: ({}+{})/{}) active from contended: {} stuck: {} completed: {}!", from_disconnected, from_exec_disconnected, no_more_work, from_prev.len(), to_execute_substage.len(), runnable_queue.task_count(), contended_count, executing_queue_count, provisioning_tracker_count, max_executing_queue_count, address_book.uncontended_task_ids.len(), address_book.stuck_tasks.len(), completed_count);
     }
 
     pub fn run(

From d0af753a8aa1cb21ee2aed387fba3e6953b63a0e Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 5 Sep 2022 12:01:53 +0900
Subject: [PATCH 2003/3199] save

---
 scheduler/src/lib.rs | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index ef14c9da7d1638..5fb6e62dafd1b1 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1362,7 +1362,7 @@ impl ScheduleStage {
                recv(from_exec) -> maybe_from_exec => {
                    if maybe_from_exec.is_err() {
                        assert_eq!(from_exec.len(), 0);
-                       from_exec_disconnected ||= true;
+                       from_exec_disconnected |= true;
                        if from_disconnected {
                            break;
                        } else {
@@ -1378,8 +1378,8 @@ impl ScheduleStage {
                recv(from_prev) -> maybe_from => {
                    if maybe_from.is_err() {
                        assert_eq!(from_prev.len(), 0);
-                       from_disconnected ||= true;
-                       no_more_work ||= runnable_queue.task_count() + contended_count + executing_queue_count + provisioning_tracker_count == 0;
+                       from_disconnected |= true;
+                       no_more_work |= runnable_queue.task_count() + contended_count + executing_queue_count + provisioning_tracker_count == 0;
                        if from_exec_disconnected || no_more_work {
                            break;
                        } else {

From 50df4dd828ecb38394415868fa49ef8f11e10a98 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 5 Sep 2022 12:07:55 +0900
Subject: [PATCH 2004/3199] save

---
 scheduler/src/lib.rs | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 5fb6e62dafd1b1..155860bb623f00 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1294,6 +1294,8 @@ impl ScheduleStage {
         from_exec: &crossbeam_channel::Receiver<UnlockablePayload>,
         maybe_to_next_stage: Option<&crossbeam_channel::Sender<PersistablePlayload>>, // assume nonblocking
     ) {
+        info!("schedule_once:initial");
+
         let mut executing_queue_count = 0_usize;
         let mut contended_count = 0;
         let mut provisioning_tracker_count = 0;

From bbde18273f77976695a9c911dc43fee4391e15ec Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 5 Sep 2022 14:23:32 +0900
Subject: [PATCH 2005/3199] save

---
 ledger/src/blockstore_processor.rs | 2 +-
 runtime/src/bank.rs                | 2 +-
 scheduler/src/lib.rs               | 2 ++
 3 files changed, 4 insertions(+), 2 deletions(-)

diff --git a/ledger/src/blockstore_processor.rs b/ledger/src/blockstore_processor.rs
index 66c29d18a7e68c..de39b0913bd562 100644
--- a/ledger/src/blockstore_processor.rs
+++ b/ledger/src/blockstore_processor.rs
@@ -194,7 +194,7 @@ fn execute_batch(
         batch,
         transaction_indexes,
     );
-    // bank().handle_unexpected_errors() -> Option<Vec<_>>
+    // bank().handle_any_unexpected_errors() -> Option<Vec<_>>
     let (tx_results, balances) = batch.bank().load_execute_and_commit_transactions(
         batch,
         MAX_PROCESSING_AGE,
diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index e02b696436c1d8..247d5033e7e88f 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1247,7 +1247,7 @@ impl Default for Scheduler {
             let mut runnable_queue = solana_scheduler::TaskQueue::default();
 
             solana_scheduler::ScheduleStage::run(
-                1,
+                123,
                 &mut runnable_queue,
                 &mut address_book,
                 &transaction_receiver,
diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 155860bb623f00..2887eda17c64b6 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1303,6 +1303,8 @@ impl ScheduleStage {
         let mut queue_clock = 0;
         let mut execute_clock = 0;
         let mut completed_count = 0_usize;
+        assert!(max_executing_queue_count > 0);
+
         let (ee_sender, ee_receiver) = crossbeam_channel::unbounded::<PersistablePlayload>();
 
         let (to_next_stage, maybe_reaper_thread_handle) = if let Some(to_next_stage) = maybe_to_next_stage {

From a4d6501f5ce914977dbf151c385e11b3d56ae7f6 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 5 Sep 2022 14:38:28 +0900
Subject: [PATCH 2006/3199] save

---
 scheduler/src/lib.rs | 18 ++++++++++++++----
 1 file changed, 14 insertions(+), 4 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 2887eda17c64b6..7fb527fb18ff15 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -35,6 +35,7 @@ pub struct ExecutionEnvironment {
     pub unique_weight: UniqueWeight,
     pub task: TaskInQueue,
     pub finalized_lock_attempts: Vec<LockAttempt>,
+    pub is_reindexed: bool,
 }
 
 impl ExecutionEnvironment {
@@ -50,7 +51,10 @@ impl ExecutionEnvironment {
     //}
     //
     #[inline(never)]
-    pub fn reindex_to_address_book(&mut self) {
+    pub fn reindex_with_address_book(&mut self) {
+        assert!(!self.is_reindexed);
+        self.is_reindexed = true;
+
         let uq = self.unique_weight;
         //self.task.trace_timestamps("in_exec(self)");
         let should_remove = self
@@ -99,6 +103,10 @@ impl ExecutionEnvironment {
                 });
         }
     }
+
+    fn is_reindexed(&self) -> bool {
+        self.is_reindexed
+    }
 }
 
 unsafe trait AtScheduleThread: Copy {}
@@ -690,7 +698,7 @@ impl Task {
     }
 
     #[inline(never)]
-    fn index_to_address_book(
+    fn index_with_address_book(
         this: &TaskInQueue,
         task_sender: &crossbeam_channel::Sender<(TaskInQueue, Vec<LockAttempt>)>,
     ) {
@@ -962,7 +970,7 @@ impl ScheduleStage {
                         next_task.mark_as_contended();
                         *contended_count = contended_count.checked_add(1).unwrap();
 
-                        Task::index_to_address_book(&next_task, task_sender);
+                        Task::index_with_address_book(&next_task, task_sender);
 
                         // maybe run lightweight prune logic on contended_queue here.
                     } else {
@@ -1217,7 +1225,9 @@ impl ScheduleStage {
     ) {
         // do par()-ly?
 
-        //ee.reindex();
+        //ee.reindex_with_address_book();
+        assert!(ee.is_reindexed());
+
         ee.task.record_commit_time(*commit_time);
         //ee.task.trace_timestamps("commit");
         //*commit_time = commit_time.checked_add(1).unwrap();

From 5034cc2f99345eabe2fe51403769477b21f6ebb4 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 5 Sep 2022 14:38:53 +0900
Subject: [PATCH 2007/3199] save

---
 scheduler/src/lib.rs | 1 +
 1 file changed, 1 insertion(+)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 7fb527fb18ff15..5fc41668c10d59 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1212,6 +1212,7 @@ impl ScheduleStage {
             unique_weight,
             cu: rng.gen_range(3, 1000),
             finalized_lock_attempts,
+            is_reindexed: Default::default(),
         })
     }
 

From 42d8248766c3b6fcb953dda95b513de054e6c6c7 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 5 Sep 2022 14:40:01 +0900
Subject: [PATCH 2008/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 5fc41668c10d59..3c585c1db98f91 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -52,7 +52,7 @@ impl ExecutionEnvironment {
     //
     #[inline(never)]
     pub fn reindex_with_address_book(&mut self) {
-        assert!(!self.is_reindexed);
+        assert!(!self.is_reindexed());
         self.is_reindexed = true;
 
         let uq = self.unique_weight;

From eced7d47b03cf21154c1a16d1933328fe32e948c Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 5 Sep 2022 15:00:23 +0900
Subject: [PATCH 2009/3199] save

---
 runtime/src/bank.rs | 23 ++++++++++++++++++++++-
 1 file changed, 22 insertions(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 247d5033e7e88f..7937911f177611 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1200,6 +1200,7 @@ impl AbiExample for BuiltinPrograms {
 struct Scheduler {
     scheduler_thread_handle: Option<std::thread::JoinHandle<Result<()>>>,
     executing_thread_handle: Option<std::thread::JoinHandle<Result<()>>>,
+    error_collector_thread_handle: Option<std::thread::JoinHandle<Result<()>>>,
     transaction_sender: Option<crossbeam_channel::Sender<solana_scheduler::SchedulablePayload>>,
     preloader: Arc<solana_scheduler::Preloader>,
     graceful_stop_initiated: bool,
@@ -1236,13 +1237,25 @@ impl Default for Scheduler {
         let (transaction_sender, transaction_receiver) = crossbeam_channel::unbounded();
         let (scheduled_ee_sender, scheduled_ee_receiver) = crossbeam_channel::unbounded();
         let (completed_ee_sender, completed_ee_receiver) = crossbeam_channel::unbounded();
+        let (droppable_ee_sender, droppable_ee_receiver) = crossbeam_channel::unbounded();
+
         let executing_thread_handle = std::thread::Builder::new().name(format!("solExec{:02}", 0)).spawn(move || {
             while let Ok(solana_scheduler::ExecutablePayload(ee)) = scheduled_ee_receiver.recv() {
+                ee.reindex_to_address_book();
                 completed_ee_sender.send(solana_scheduler::UnlockablePayload(ee)).unwrap();
             }
             Ok(())
         }).unwrap();
 
+        let error_collector_thread_handle = std::thread::Builder::new().name(format!("solErrorCol{:02}"r 0)).spawn(move || {
+            while let Ok(solana_scheduler::DroppablePayload(ee)) = droppable_ee_receiver.recv() {
+                if false /* ee.is_aborted() */ {
+                    errors.push(ee)
+                }
+            }
+            Ok(())
+        }).unwrap();
+
         let scheduler_thread_handle = std::thread::Builder::new().name("solScheduler".to_string()).spawn(move || {
             let mut runnable_queue = solana_scheduler::TaskQueue::default();
 
@@ -1253,7 +1266,7 @@ impl Default for Scheduler {
                 &transaction_receiver,
                 &scheduled_ee_sender,
                 &completed_ee_receiver,
-                None,
+                Some(&droppable_ee_sender),
             );
             drop(transaction_receiver);
             drop(scheduled_ee_sender);
@@ -1265,6 +1278,7 @@ impl Default for Scheduler {
         Self {
             scheduler_thread_handle: Some(scheduler_thread_handle),
             executing_thread_handle: Some(executing_thread_handle),
+            error_collector_thread_handle: Some(error_collector_thread_handle),
             transaction_sender: Some(transaction_sender),
             preloader,
             graceful_stop_initiated: Default::default(),
@@ -6321,6 +6335,13 @@ impl Bank {
         }
     }
 
+    pub fn handle_aborted_transactions(
+        &self,
+    ) -> Vec<Result<()> {
+        let scheduler = self.scheduler.read().unwrap();
+        scheduler.handle_aborted_executions()
+    }
+
     /// Process a batch of transactions.
     #[must_use]
     pub fn load_execute_and_commit_transactions(

From 0e5f1e93ee0ad57d1c7f744a876ff5fa0ad9739c Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 5 Sep 2022 15:00:32 +0900
Subject: [PATCH 2010/3199] save

---
 scheduler/src/lib.rs | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 3c585c1db98f91..2318e49ba1cdc4 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1303,7 +1303,7 @@ impl ScheduleStage {
         from_prev: &crossbeam_channel::Receiver<SchedulablePayload>,
         to_execute_substage: &crossbeam_channel::Sender<ExecutablePayload>,
         from_exec: &crossbeam_channel::Receiver<UnlockablePayload>,
-        maybe_to_next_stage: Option<&crossbeam_channel::Sender<PersistablePlayload>>, // assume nonblocking
+        maybe_to_next_stage: Option<&crossbeam_channel::Sender<DroppablePayload>>, // assume nonblocking
     ) {
         info!("schedule_once:initial");
 
@@ -1316,7 +1316,7 @@ impl ScheduleStage {
         let mut completed_count = 0_usize;
         assert!(max_executing_queue_count > 0);
 
-        let (ee_sender, ee_receiver) = crossbeam_channel::unbounded::<PersistablePlayload>();
+        let (ee_sender, ee_receiver) = crossbeam_channel::unbounded::<DroppablePayload>();
 
         let (to_next_stage, maybe_reaper_thread_handle) = if let Some(to_next_stage) = maybe_to_next_stage {
             (to_next_stage, None)
@@ -1329,7 +1329,7 @@ impl ScheduleStage {
                     unsafe impl NotAtScheduleThread for NotAtTopOfScheduleThread {}
                     let nast = NotAtTopOfScheduleThread;
 
-                    while let Ok(PersistablePlayload(mut a)) = ee_receiver.recv() {
+                    while let Ok(DroppablePayload(mut a)) = ee_receiver.recv() {
                         assert!(a.task.lock_attempts_not_mut(nast).is_empty());
                         //assert!(a.task.sequence_time() != usize::max_value());
                         //let lock_attempts = std::mem::take(&mut a.lock_attempts);
@@ -1388,7 +1388,7 @@ impl ScheduleStage {
                     executing_queue_count = executing_queue_count.checked_sub(1).unwrap();
                     completed_count = completed_count.checked_add(1).unwrap();
                     Self::commit_completed_execution(ast, &mut processed_execution_environment, address_book, &mut execute_clock, &mut provisioning_tracker_count);
-                    to_next_stage.send(PersistablePlayload(processed_execution_environment)).unwrap();
+                    to_next_stage.send(DroppablePayload(processed_execution_environment)).unwrap();
                }
                recv(from_prev) -> maybe_from => {
                    if maybe_from.is_err() {
@@ -1468,7 +1468,7 @@ impl ScheduleStage {
                             &mut execute_clock,
                             &mut provisioning_tracker_count,
                         );
-                        to_next_stage.send(PersistablePlayload(processed_execution_environment)).unwrap();
+                        to_next_stage.send(DroppablePayload(processed_execution_environment)).unwrap();
                     }
                     if !empty_from {
                         let task = from_prev.recv().unwrap().0;
@@ -1501,7 +1501,7 @@ impl ScheduleStage {
         from: &crossbeam_channel::Receiver<SchedulablePayload>,
         to_execute_substage: &crossbeam_channel::Sender<ExecutablePayload>,
         from_execute_substage: &crossbeam_channel::Receiver<UnlockablePayload>,
-        maybe_to_next_stage: Option<&crossbeam_channel::Sender<PersistablePlayload>>, // assume nonblocking
+        maybe_to_next_stage: Option<&crossbeam_channel::Sender<DroppablePayload>>, // assume nonblocking
     ) {
         #[derive(Clone, Copy, Debug)]
         struct AtTopOfScheduleThread;
@@ -1523,7 +1523,7 @@ impl ScheduleStage {
 pub struct SchedulablePayload(pub TaskInQueue);
 pub struct ExecutablePayload(pub Box<ExecutionEnvironment>);
 pub struct UnlockablePayload(pub Box<ExecutionEnvironment>);
-pub struct PersistablePlayload(pub Box<ExecutionEnvironment>);
+pub struct DroppablePayload(pub Box<ExecutionEnvironment>);
 
 struct ExecuteStage {
     //bank: Bank,

From 9861405042f2766c0557ed680faff1143b5600f0 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 5 Sep 2022 15:01:03 +0900
Subject: [PATCH 2011/3199] save

---
 runtime/src/bank.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 7937911f177611..fa6680b10d916c 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -6337,7 +6337,7 @@ impl Bank {
 
     pub fn handle_aborted_transactions(
         &self,
-    ) -> Vec<Result<()> {
+    ) -> Vec<Result<()>> {
         let scheduler = self.scheduler.read().unwrap();
         scheduler.handle_aborted_executions()
     }

From e527a8d2ba52140cb07af9b4b5239c934f581ba7 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 5 Sep 2022 15:01:16 +0900
Subject: [PATCH 2012/3199] save

---
 runtime/src/bank.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index fa6680b10d916c..7dbbcce80c0f44 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1247,7 +1247,7 @@ impl Default for Scheduler {
             Ok(())
         }).unwrap();
 
-        let error_collector_thread_handle = std::thread::Builder::new().name(format!("solErrorCol{:02}"r 0)).spawn(move || {
+        let error_collector_thread_handle = std::thread::Builder::new().name(format!("solErrorCol{:02}", 0)).spawn(move || {
             while let Ok(solana_scheduler::DroppablePayload(ee)) = droppable_ee_receiver.recv() {
                 if false /* ee.is_aborted() */ {
                     errors.push(ee)

From 0b8ea7367f79b247bf37850e16f7e2f45df80a9b Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 5 Sep 2022 15:01:47 +0900
Subject: [PATCH 2013/3199] save

---
 runtime/src/bank.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 7dbbcce80c0f44..524ea28943fb9b 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1241,7 +1241,7 @@ impl Default for Scheduler {
 
         let executing_thread_handle = std::thread::Builder::new().name(format!("solExec{:02}", 0)).spawn(move || {
             while let Ok(solana_scheduler::ExecutablePayload(ee)) = scheduled_ee_receiver.recv() {
-                ee.reindex_to_address_book();
+                ee.reindex_with_address_book();
                 completed_ee_sender.send(solana_scheduler::UnlockablePayload(ee)).unwrap();
             }
             Ok(())

From 392dfc6162dac521d625a6e688c219d9d1f65715 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 5 Sep 2022 15:03:14 +0900
Subject: [PATCH 2014/3199] save

---
 runtime/src/bank.rs | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 524ea28943fb9b..6c767ae7b18092 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1204,6 +1204,7 @@ struct Scheduler {
     transaction_sender: Option<crossbeam_channel::Sender<solana_scheduler::SchedulablePayload>>,
     preloader: Arc<solana_scheduler::Preloader>,
     graceful_stop_initiated: bool,
+    errors: Vec<Result<()>>,
 }
 
 impl Scheduler {
@@ -1250,7 +1251,7 @@ impl Default for Scheduler {
         let error_collector_thread_handle = std::thread::Builder::new().name(format!("solErrorCol{:02}", 0)).spawn(move || {
             while let Ok(solana_scheduler::DroppablePayload(ee)) = droppable_ee_receiver.recv() {
                 if false /* ee.is_aborted() */ {
-                    errors.push(ee)
+                    errors.push(Ok(()))
                 }
             }
             Ok(())

From fbd449e1395ff0366784c25487e9fe961f1955f7 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 5 Sep 2022 15:03:52 +0900
Subject: [PATCH 2015/3199] save

---
 runtime/src/bank.rs | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 6c767ae7b18092..aaf2efc6e549ee 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1204,7 +1204,7 @@ struct Scheduler {
     transaction_sender: Option<crossbeam_channel::Sender<solana_scheduler::SchedulablePayload>>,
     preloader: Arc<solana_scheduler::Preloader>,
     graceful_stop_initiated: bool,
-    errors: Vec<Result<()>>,
+    errors: Mutex<Vec<Result<()>>>,
 }
 
 impl Scheduler {
@@ -1248,6 +1248,8 @@ impl Default for Scheduler {
             Ok(())
         }).unwrap();
 
+        let errors = std::Mutex::new(Vec::new());
+
         let error_collector_thread_handle = std::thread::Builder::new().name(format!("solErrorCol{:02}", 0)).spawn(move || {
             while let Ok(solana_scheduler::DroppablePayload(ee)) = droppable_ee_receiver.recv() {
                 if false /* ee.is_aborted() */ {
@@ -1283,6 +1285,7 @@ impl Default for Scheduler {
             transaction_sender: Some(transaction_sender),
             preloader,
             graceful_stop_initiated: Default::default(),
+            errors,
         }
     }
 }

From 05c2554bd47603831134988a259020b2e37b5ed8 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 5 Sep 2022 15:04:03 +0900
Subject: [PATCH 2016/3199] save

---
 runtime/src/bank.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index aaf2efc6e549ee..f0bf5dbb9654d3 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1204,7 +1204,7 @@ struct Scheduler {
     transaction_sender: Option<crossbeam_channel::Sender<solana_scheduler::SchedulablePayload>>,
     preloader: Arc<solana_scheduler::Preloader>,
     graceful_stop_initiated: bool,
-    errors: Mutex<Vec<Result<()>>>,
+    errors: std::sync::Mutex<Vec<Result<()>>>,
 }
 
 impl Scheduler {

From 4cf92e7be5ba96b976d4b70faf8e93c0930257b5 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 5 Sep 2022 15:04:21 +0900
Subject: [PATCH 2017/3199] save

---
 runtime/src/bank.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index f0bf5dbb9654d3..1c6a286144d4df 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1248,7 +1248,7 @@ impl Default for Scheduler {
             Ok(())
         }).unwrap();
 
-        let errors = std::Mutex::new(Vec::new());
+        let errors = std::sync::Mutex::new(Vec::new());
 
         let error_collector_thread_handle = std::thread::Builder::new().name(format!("solErrorCol{:02}", 0)).spawn(move || {
             while let Ok(solana_scheduler::DroppablePayload(ee)) = droppable_ee_receiver.recv() {

From c965d58235c1526d47e32c193f0ae4a9d1e70b09 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 5 Sep 2022 15:04:40 +0900
Subject: [PATCH 2018/3199] save

---
 runtime/src/bank.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 1c6a286144d4df..1783809ccd9091 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1253,7 +1253,7 @@ impl Default for Scheduler {
         let error_collector_thread_handle = std::thread::Builder::new().name(format!("solErrorCol{:02}", 0)).spawn(move || {
             while let Ok(solana_scheduler::DroppablePayload(ee)) = droppable_ee_receiver.recv() {
                 if false /* ee.is_aborted() */ {
-                    errors.push(Ok(()))
+                    errors.lock().unwrap().push(Ok(()))
                 }
             }
             Ok(())

From d2fe81175d12ee9e9147d7176935720db612ab85 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 5 Sep 2022 15:06:27 +0900
Subject: [PATCH 2019/3199] save

---
 runtime/src/bank.rs | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 1783809ccd9091..828eab3b9eefe0 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1308,6 +1308,10 @@ impl Scheduler {
 
         Ok(())
     }
+
+    fn handle_aborted_executions(&self) -> Vec<Result<()>> {
+        self.lock().unwrap().take()
+    }
 }
 
 impl Drop for Scheduler {

From 6db7ee2116f19eee6bf6f959706582477987fc5a Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 5 Sep 2022 15:06:44 +0900
Subject: [PATCH 2020/3199] save

---
 runtime/src/bank.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 828eab3b9eefe0..257430131af773 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1310,7 +1310,7 @@ impl Scheduler {
     }
 
     fn handle_aborted_executions(&self) -> Vec<Result<()>> {
-        self.lock().unwrap().take()
+        self.errors.lock().unwrap().take()
     }
 }
 

From 9c46bb9e8d16794357b13cbffdcd2b83dfcfd90c Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 5 Sep 2022 15:07:27 +0900
Subject: [PATCH 2021/3199] save

---
 runtime/src/bank.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 257430131af773..5d8bcc6c729ae4 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1310,7 +1310,7 @@ impl Scheduler {
     }
 
     fn handle_aborted_executions(&self) -> Vec<Result<()>> {
-        self.errors.lock().unwrap().take()
+        std::mem::take(&mut self.errors.lock().unwrap())
     }
 }
 

From 763dec9ca975f339fff331c516f30b5da4d600bc Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 5 Sep 2022 15:08:15 +0900
Subject: [PATCH 2022/3199] save

---
 runtime/src/bank.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 5d8bcc6c729ae4..90b531b9aa5bfa 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1241,7 +1241,7 @@ impl Default for Scheduler {
         let (droppable_ee_sender, droppable_ee_receiver) = crossbeam_channel::unbounded();
 
         let executing_thread_handle = std::thread::Builder::new().name(format!("solExec{:02}", 0)).spawn(move || {
-            while let Ok(solana_scheduler::ExecutablePayload(ee)) = scheduled_ee_receiver.recv() {
+            while let Ok(solana_scheduler::ExecutablePayload(mut ee)) = scheduled_ee_receiver.recv() {
                 ee.reindex_with_address_book();
                 completed_ee_sender.send(solana_scheduler::UnlockablePayload(ee)).unwrap();
             }
@@ -1299,7 +1299,7 @@ impl Scheduler {
         self.graceful_stop_initiated = true;
 
         info!("Scheduler::gracefully_stop(): waiting..");
-        let mut transaction_sender = self.transaction_sender.take().unwrap();
+        let transaction_sender = self.transaction_sender.take().unwrap();
         drop(transaction_sender);
         let h = self.executing_thread_handle.take().unwrap();
         h.join().unwrap()?;

From b378745ebe806fae2a8d3f67c54c4d718157d7e8 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 5 Sep 2022 15:09:26 +0900
Subject: [PATCH 2023/3199] save

---
 runtime/src/bank.rs | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 90b531b9aa5bfa..acb7e4f96e9411 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1248,9 +1248,10 @@ impl Default for Scheduler {
             Ok(())
         }).unwrap();
 
-        let errors = std::sync::Mutex::new(Vec::new());
+        let errors = Arc::new(std::sync::Mutex::new(Vec::new()));
 
         let error_collector_thread_handle = std::thread::Builder::new().name(format!("solErrorCol{:02}", 0)).spawn(move || {
+            let errors = errors.clone();
             while let Ok(solana_scheduler::DroppablePayload(ee)) = droppable_ee_receiver.recv() {
                 if false /* ee.is_aborted() */ {
                     errors.lock().unwrap().push(Ok(()))

From 365ca62ab7e40ac843f81d057abded9f065f5a4e Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 5 Sep 2022 15:09:57 +0900
Subject: [PATCH 2024/3199] save

---
 runtime/src/bank.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index acb7e4f96e9411..9e2e0d17edaf6e 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1204,7 +1204,7 @@ struct Scheduler {
     transaction_sender: Option<crossbeam_channel::Sender<solana_scheduler::SchedulablePayload>>,
     preloader: Arc<solana_scheduler::Preloader>,
     graceful_stop_initiated: bool,
-    errors: std::sync::Mutex<Vec<Result<()>>>,
+    errors: Arc<std::sync::Mutex<Vec<Result<()>>>>,
 }
 
 impl Scheduler {

From 35401eef6a7e271e5354f24c02b1837bbc6ffbac Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 5 Sep 2022 15:10:50 +0900
Subject: [PATCH 2025/3199] save

---
 runtime/src/bank.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 9e2e0d17edaf6e..36dbe7cdb61b8d 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1249,12 +1249,12 @@ impl Default for Scheduler {
         }).unwrap();
 
         let errors = Arc::new(std::sync::Mutex::new(Vec::new()));
+        let errors_in_collector_thread = Arc::clone(errors);
 
         let error_collector_thread_handle = std::thread::Builder::new().name(format!("solErrorCol{:02}", 0)).spawn(move || {
-            let errors = errors.clone();
             while let Ok(solana_scheduler::DroppablePayload(ee)) = droppable_ee_receiver.recv() {
                 if false /* ee.is_aborted() */ {
-                    errors.lock().unwrap().push(Ok(()))
+                    errors_in_collector_thread.lock().unwrap().push(Ok(()))
                 }
             }
             Ok(())

From 5b8da9e3abc7b601401fa9198a1a9a79da042ee4 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 5 Sep 2022 15:11:04 +0900
Subject: [PATCH 2026/3199] save

---
 runtime/src/bank.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 36dbe7cdb61b8d..c304409397202e 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1249,7 +1249,7 @@ impl Default for Scheduler {
         }).unwrap();
 
         let errors = Arc::new(std::sync::Mutex::new(Vec::new()));
-        let errors_in_collector_thread = Arc::clone(errors);
+        let errors_in_collector_thread = Arc::clone(&errors);
 
         let error_collector_thread_handle = std::thread::Builder::new().name(format!("solErrorCol{:02}", 0)).spawn(move || {
             while let Ok(solana_scheduler::DroppablePayload(ee)) = droppable_ee_receiver.recv() {

From 653adfb711054d6642a7fed0f167f1394ccbb147 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 5 Sep 2022 15:11:43 +0900
Subject: [PATCH 2027/3199] save

---
 runtime/src/bank.rs | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index c304409397202e..16974f0753f354 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1252,7 +1252,7 @@ impl Default for Scheduler {
         let errors_in_collector_thread = Arc::clone(&errors);
 
         let error_collector_thread_handle = std::thread::Builder::new().name(format!("solErrorCol{:02}", 0)).spawn(move || {
-            while let Ok(solana_scheduler::DroppablePayload(ee)) = droppable_ee_receiver.recv() {
+            while let Ok(solana_scheduler::DroppablePayload(_ee)) = droppable_ee_receiver.recv() {
                 if false /* ee.is_aborted() */ {
                     errors_in_collector_thread.lock().unwrap().push(Ok(()))
                 }
@@ -1306,6 +1306,8 @@ impl Scheduler {
         h.join().unwrap()?;
         let h = self.scheduler_thread_handle.take().unwrap();
         h.join().unwrap()?;
+        let h = self.error_collector_thread_handle.take().unwrap();
+        h.join().unwrap()?;
 
         Ok(())
     }

From fbb052f90811e79fb4250d0c3c621b5c15ff5211 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 5 Sep 2022 15:25:37 +0900
Subject: [PATCH 2028/3199] save

---
 ledger/src/blockstore_processor.rs | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/ledger/src/blockstore_processor.rs b/ledger/src/blockstore_processor.rs
index de39b0913bd562..b52473c5ee4323 100644
--- a/ledger/src/blockstore_processor.rs
+++ b/ledger/src/blockstore_processor.rs
@@ -194,7 +194,10 @@ fn execute_batch(
         batch,
         transaction_indexes,
     );
-    // bank().handle_any_unexpected_errors() -> Option<Vec<_>>
+    if let Some(first_error_from_scheduler) = batch.bank().handle_aborted_transactions().iter().first() {
+        first_error_from_scheduler?
+    }
+
     let (tx_results, balances) = batch.bank().load_execute_and_commit_transactions(
         batch,
         MAX_PROCESSING_AGE,

From cefcfebc8cda8836d6b13928ba09f0169b3609ed Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 5 Sep 2022 15:28:27 +0900
Subject: [PATCH 2029/3199] save

---
 ledger/src/blockstore_processor.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ledger/src/blockstore_processor.rs b/ledger/src/blockstore_processor.rs
index b52473c5ee4323..7c27fcffeee552 100644
--- a/ledger/src/blockstore_processor.rs
+++ b/ledger/src/blockstore_processor.rs
@@ -194,7 +194,7 @@ fn execute_batch(
         batch,
         transaction_indexes,
     );
-    if let Some(first_error_from_scheduler) = batch.bank().handle_aborted_transactions().iter().first() {
+    if let Some(first_error_from_scheduler) = batch.bank().handle_aborted_transactions().iter().next() {
         first_error_from_scheduler?
     }
 

From b3369c943a991994733d18241e7a08dac109671d Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 5 Sep 2022 15:31:20 +0900
Subject: [PATCH 2030/3199] save

---
 ledger/src/blockstore_processor.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ledger/src/blockstore_processor.rs b/ledger/src/blockstore_processor.rs
index 7c27fcffeee552..a5306bc32c6d0e 100644
--- a/ledger/src/blockstore_processor.rs
+++ b/ledger/src/blockstore_processor.rs
@@ -194,7 +194,7 @@ fn execute_batch(
         batch,
         transaction_indexes,
     );
-    if let Some(first_error_from_scheduler) = batch.bank().handle_aborted_transactions().iter().next() {
+    if let Some(first_error_from_scheduler) = batch.bank().handle_aborted_transactions().into_iter().next() {
         first_error_from_scheduler?
     }
 

From 5d6a37a904eb13dd813b6dfc8ffc16c6ec81e3b0 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 5 Sep 2022 16:16:14 +0900
Subject: [PATCH 2031/3199] save

---
 runtime/src/bank.rs                     | 16 ++++++++++------
 scheduler/src/lib.rs                    |  5 +++--
 transaction-scheduler-bench/src/main.rs |  2 +-
 3 files changed, 14 insertions(+), 9 deletions(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 16974f0753f354..acaea8ec00acb6 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1199,7 +1199,7 @@ impl AbiExample for BuiltinPrograms {
 #[derive(Debug)]
 struct Scheduler {
     scheduler_thread_handle: Option<std::thread::JoinHandle<Result<()>>>,
-    executing_thread_handle: Option<std::thread::JoinHandle<Result<()>>>,
+    executing_thread_handles: Option<Vec<std::thread::JoinHandle<Result<()>>>>,
     error_collector_thread_handle: Option<std::thread::JoinHandle<Result<()>>>,
     transaction_sender: Option<crossbeam_channel::Sender<solana_scheduler::SchedulablePayload>>,
     preloader: Arc<solana_scheduler::Preloader>,
@@ -1240,13 +1240,16 @@ impl Default for Scheduler {
         let (completed_ee_sender, completed_ee_receiver) = crossbeam_channel::unbounded();
         let (droppable_ee_sender, droppable_ee_receiver) = crossbeam_channel::unbounded();
 
-        let executing_thread_handle = std::thread::Builder::new().name(format!("solExec{:02}", 0)).spawn(move || {
+        let executing_thread_handles = (0..8).map(|thx| {
+            let (scheduled_ee_receiver, completed_ee_sender) = (scheduled_ee_receiver.clone(), completed_ee_sender.clone());
+
+            std::thread::Builder::new().name(format!("solExec{:02}", thx)).spawn(move || {
             while let Ok(solana_scheduler::ExecutablePayload(mut ee)) = scheduled_ee_receiver.recv() {
                 ee.reindex_with_address_book();
                 completed_ee_sender.send(solana_scheduler::UnlockablePayload(ee)).unwrap();
             }
             Ok(())
-        }).unwrap();
+        }).unwrap()}).collect();
 
         let errors = Arc::new(std::sync::Mutex::new(Vec::new()));
         let errors_in_collector_thread = Arc::clone(&errors);
@@ -1281,7 +1284,7 @@ impl Default for Scheduler {
 
         Self {
             scheduler_thread_handle: Some(scheduler_thread_handle),
-            executing_thread_handle: Some(executing_thread_handle),
+            executing_thread_handles: Some(executing_thread_handles),
             error_collector_thread_handle: Some(error_collector_thread_handle),
             transaction_sender: Some(transaction_sender),
             preloader,
@@ -1302,8 +1305,9 @@ impl Scheduler {
         info!("Scheduler::gracefully_stop(): waiting..");
         let transaction_sender = self.transaction_sender.take().unwrap();
         drop(transaction_sender);
-        let h = self.executing_thread_handle.take().unwrap();
-        h.join().unwrap()?;
+        for executing_thread_handle in self.executing_thread_handles.take().unwrap() {
+            executing_thread_handle.join().unwrap()?;
+        }
         let h = self.scheduler_thread_handle.take().unwrap();
         h.join().unwrap()?;
         let h = self.error_collector_thread_handle.take().unwrap();
diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 2318e49ba1cdc4..b5a1d91d31a84c 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1305,7 +1305,8 @@ impl ScheduleStage {
         from_exec: &crossbeam_channel::Receiver<UnlockablePayload>,
         maybe_to_next_stage: Option<&crossbeam_channel::Sender<DroppablePayload>>, // assume nonblocking
     ) {
-        info!("schedule_once:initial");
+        let random_id = rand::thread_rng().gen::<u64>();
+        info!("schedule_once:initial id_{:016x}", random_id);
 
         let mut executing_queue_count = 0_usize;
         let mut contended_count = 0;
@@ -1491,7 +1492,7 @@ impl ScheduleStage {
             indexer_handle.join().unwrap().unwrap();
         }
 
-        info!("schedule_once:final(from_disconnected: {}, from_exec_disconnected: {}, no_more_work: {}) (from: {}, to: {}, runnnable: {}, contended: {}, (immediate+provisional)/max: ({}+{})/{}) active from contended: {} stuck: {} completed: {}!", from_disconnected, from_exec_disconnected, no_more_work, from_prev.len(), to_execute_substage.len(), runnable_queue.task_count(), contended_count, executing_queue_count, provisioning_tracker_count, max_executing_queue_count, address_book.uncontended_task_ids.len(), address_book.stuck_tasks.len(), completed_count);
+        info!("schedule_once:final id_{:016x} (from_disconnected: {}, from_exec_disconnected: {}, no_more_work: {}) (from: {}, to: {}, runnnable: {}, contended: {}, (immediate+provisional)/max: ({}+{})/{}) active from contended: {} stuck: {} completed: {}!", random_id, from_disconnected, from_exec_disconnected, no_more_work, from_prev.len(), to_execute_substage.len(), runnable_queue.task_count(), contended_count, executing_queue_count, provisioning_tracker_count, max_executing_queue_count, address_book.uncontended_task_ids.len(), address_book.stuck_tasks.len(), completed_count);
     }
 
     pub fn run(
diff --git a/transaction-scheduler-bench/src/main.rs b/transaction-scheduler-bench/src/main.rs
index 5f493bc65f4c25..ee1dcb1439abaa 100644
--- a/transaction-scheduler-bench/src/main.rs
+++ b/transaction-scheduler-bench/src/main.rs
@@ -360,7 +360,7 @@ fn handle_transaction_batch(
         .priority_collected
         .fetch_add(priority_collected, Ordering::Relaxed);
 
-    transaction_batch.0.reindex_to_address_book();
+    transaction_batch.0.reindex_with_address_book();
     completed_transaction_sender
         .send(solana_scheduler::UnlockablePayload(transaction_batch.0))
         .unwrap();

From d3d5d88b30da588d3dd1ed53ac294ceb7d7f8a38 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 5 Sep 2022 16:32:43 +0900
Subject: [PATCH 2032/3199] save

---
 scheduler/src/lib.rs | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index b5a1d91d31a84c..dc35c6e5644497 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -29,13 +29,14 @@ unsafe impl Sync for PageRc {}
 type CU = u64;
 
 #[derive(Debug)]
-pub struct ExecutionEnvironment {
+pub struct ExecutionEnvironment<T> {
     //accounts: Vec<i8>,
     pub cu: CU,
     pub unique_weight: UniqueWeight,
     pub task: TaskInQueue,
     pub finalized_lock_attempts: Vec<LockAttempt>,
     pub is_reindexed: bool,
+    pub extra: T,
 }
 
 impl ExecutionEnvironment {

From 9190966b946e33ea50c0f2bc03960fee7e5dec0e Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 5 Sep 2022 16:33:11 +0900
Subject: [PATCH 2033/3199] save

---
 scheduler/src/lib.rs | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index dc35c6e5644497..212fb811174014 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1523,9 +1523,9 @@ impl ScheduleStage {
 }
 
 pub struct SchedulablePayload(pub TaskInQueue);
-pub struct ExecutablePayload(pub Box<ExecutionEnvironment>);
-pub struct UnlockablePayload(pub Box<ExecutionEnvironment>);
-pub struct DroppablePayload(pub Box<ExecutionEnvironment>);
+pub struct ExecutablePayload<T>(pub Box<ExecutionEnvironment<T>>);
+pub struct UnlockablePayload<T>(pub Box<ExecutionEnvironment<T>>);
+pub struct DroppablePayload<T>(pub Box<ExecutionEnvironment<T>>);
 
 struct ExecuteStage {
     //bank: Bank,

From 42d8583034878999bed8e55991a6d681bedc7b33 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 5 Sep 2022 16:33:32 +0900
Subject: [PATCH 2034/3199] save

---
 scheduler/src/lib.rs | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 212fb811174014..fefe3db5277278 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1496,14 +1496,14 @@ impl ScheduleStage {
         info!("schedule_once:final id_{:016x} (from_disconnected: {}, from_exec_disconnected: {}, no_more_work: {}) (from: {}, to: {}, runnnable: {}, contended: {}, (immediate+provisional)/max: ({}+{})/{}) active from contended: {} stuck: {} completed: {}!", random_id, from_disconnected, from_exec_disconnected, no_more_work, from_prev.len(), to_execute_substage.len(), runnable_queue.task_count(), contended_count, executing_queue_count, provisioning_tracker_count, max_executing_queue_count, address_book.uncontended_task_ids.len(), address_book.stuck_tasks.len(), completed_count);
     }
 
-    pub fn run(
+    pub fn run<T>(
         max_executing_queue_count: usize,
         runnable_queue: &mut TaskQueue,
         address_book: &mut AddressBook,
-        from: &crossbeam_channel::Receiver<SchedulablePayload>,
-        to_execute_substage: &crossbeam_channel::Sender<ExecutablePayload>,
-        from_execute_substage: &crossbeam_channel::Receiver<UnlockablePayload>,
-        maybe_to_next_stage: Option<&crossbeam_channel::Sender<DroppablePayload>>, // assume nonblocking
+        from: &crossbeam_channel::Receiver<SchedulablePayload<T>>,
+        to_execute_substage: &crossbeam_channel::Sender<ExecutablePayload<T>>,
+        from_execute_substage: &crossbeam_channel::Receiver<UnlockablePayload<T>>,
+        maybe_to_next_stage: Option<&crossbeam_channel::Sender<DroppablePayload<T>>>, // assume nonblocking
     ) {
         #[derive(Clone, Copy, Debug)]
         struct AtTopOfScheduleThread;

From 84648c882f409a61c9d45a251cb6c52d3117476a Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 5 Sep 2022 16:34:01 +0900
Subject: [PATCH 2035/3199] save

---
 scheduler/src/lib.rs | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index fefe3db5277278..6e0c6c5e9e1001 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1296,15 +1296,15 @@ impl ScheduleStage {
         Self::push_to_runnable_queue(weighted_tx, runnable_queue)
     }
 
-    fn _run<AST: AtScheduleThread>(
+    fn _run<AST: AtScheduleThread, T>(
         ast: AST,
         max_executing_queue_count: usize,
         runnable_queue: &mut TaskQueue,
         address_book: &mut AddressBook,
-        from_prev: &crossbeam_channel::Receiver<SchedulablePayload>,
-        to_execute_substage: &crossbeam_channel::Sender<ExecutablePayload>,
-        from_exec: &crossbeam_channel::Receiver<UnlockablePayload>,
-        maybe_to_next_stage: Option<&crossbeam_channel::Sender<DroppablePayload>>, // assume nonblocking
+        from_prev: &crossbeam_channel::Receiver<SchedulablePayload<T>>,
+        to_execute_substage: &crossbeam_channel::Sender<ExecutablePayload<T>>,
+        from_exec: &crossbeam_channel::Receiver<UnlockablePayload<T>>,
+        maybe_to_next_stage: Option<&crossbeam_channel::Sender<DroppablePayload<T>>>, // assume nonblocking
     ) {
         let random_id = rand::thread_rng().gen::<u64>();
         info!("schedule_once:initial id_{:016x}", random_id);

From 34d54d56cd261e8e55a0f86179c5ff4683d61cb7 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 5 Sep 2022 16:35:02 +0900
Subject: [PATCH 2036/3199] save

---
 scheduler/src/lib.rs | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 6e0c6c5e9e1001..9bd8573baba93c 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -499,7 +499,7 @@ struct Bundle {
 }
 
 #[derive(Debug)]
-pub struct Task {
+pub struct Task<T> {
     unique_weight: UniqueWeight,
     pub tx: (SanitizedTransaction, LockAttemptsInCell), // actually should be Bundle
     pub contention_count: std::sync::atomic::AtomicUsize,
@@ -512,6 +512,7 @@ pub struct Task {
     pub execute_time: std::sync::atomic::AtomicUsize,
     pub commit_time: std::sync::atomic::AtomicUsize,
     pub for_indexer: LockAttemptsInCell,
+    pub extra: T,
 }
 
 #[derive(Debug)]
@@ -730,7 +731,7 @@ pub struct TaskQueue {
     //tasks: std::sync::Arc<dashmap::DashMap<UniqueWeight, TaskInQueue>>,
 }
 
-pub type TaskInQueue = triomphe::Arc<Task>;
+pub type TaskInQueue = triomphe::Arc<Task<T>>;
 
 //type TaskQueueEntry<'a> = im::ordmap::Entry<'a, UniqueWeight, TaskInQueue>;
 //type TaskQueueOccupiedEntry<'a> = im::ordmap::OccupiedEntry<'a, UniqueWeight, TaskInQueue>;
@@ -1522,7 +1523,7 @@ impl ScheduleStage {
     }
 }
 
-pub struct SchedulablePayload(pub TaskInQueue);
+pub struct SchedulablePayload(pub TaskInQueue<T>);
 pub struct ExecutablePayload<T>(pub Box<ExecutionEnvironment<T>>);
 pub struct UnlockablePayload<T>(pub Box<ExecutionEnvironment<T>>);
 pub struct DroppablePayload<T>(pub Box<ExecutionEnvironment<T>>);

From 54be111f641cc0e2aaf60954c2286462a27052fa Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 5 Sep 2022 16:35:22 +0900
Subject: [PATCH 2037/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 9bd8573baba93c..9bf0d4e181afa5 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1523,7 +1523,7 @@ impl ScheduleStage {
     }
 }
 
-pub struct SchedulablePayload(pub TaskInQueue<T>);
+pub struct SchedulablePayload<T>(pub TaskInQueue<T>);
 pub struct ExecutablePayload<T>(pub Box<ExecutionEnvironment<T>>);
 pub struct UnlockablePayload<T>(pub Box<ExecutionEnvironment<T>>);
 pub struct DroppablePayload<T>(pub Box<ExecutionEnvironment<T>>);

From ac30cfacb5fc57b62b07bc3563f9ddaa2d087bb7 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 5 Sep 2022 16:35:45 +0900
Subject: [PATCH 2038/3199] save

---
 scheduler/src/lib.rs | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 9bf0d4e181afa5..58a4aa692d4ad0 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1257,9 +1257,9 @@ impl ScheduleStage {
     }
 
     #[inline(never)]
-    fn schedule_next_execution<AST: AtScheduleThread>(
+    fn schedule_next_execution<AST: AtScheduleThread, T>(
         ast: AST,
-        task_sender: &crossbeam_channel::Sender<(TaskInQueue, Vec<LockAttempt>)>,
+        task_sender: &crossbeam_channel::Sender<(TaskInQueue<T>, Vec<LockAttempt>)>,
         runnable_queue: &mut TaskQueue,
         address_book: &mut AddressBook,
         contended_count: &mut usize,
@@ -1268,7 +1268,7 @@ impl ScheduleStage {
         queue_clock: &mut usize,
         execute_clock: &mut usize,
         provisioning_tracker_count: &mut usize,
-    ) -> Option<Box<ExecutionEnvironment>> {
+    ) -> Option<Box<ExecutionEnvironment<T>>> {
         let maybe_ee = Self::pop_from_queue_then_lock(
             ast,
             task_sender,

From 6e3d2d58092467c35732b23f9bbe0b4b8ee9b5c7 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 5 Sep 2022 16:35:57 +0900
Subject: [PATCH 2039/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 58a4aa692d4ad0..5f649251dddba9 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -731,7 +731,7 @@ pub struct TaskQueue {
     //tasks: std::sync::Arc<dashmap::DashMap<UniqueWeight, TaskInQueue>>,
 }
 
-pub type TaskInQueue = triomphe::Arc<Task<T>>;
+pub type TaskInQueue<T> = triomphe::Arc<Task<T>>;
 
 //type TaskQueueEntry<'a> = im::ordmap::Entry<'a, UniqueWeight, TaskInQueue>;
 //type TaskQueueOccupiedEntry<'a> = im::ordmap::OccupiedEntry<'a, UniqueWeight, TaskInQueue>;

From a7c341bedb51644e0f57bbc4affea23e45977bb6 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 5 Sep 2022 16:36:24 +0900
Subject: [PATCH 2040/3199] save

---
 scheduler/src/lib.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 5f649251dddba9..65cb2b3599db4d 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1287,8 +1287,8 @@ impl ScheduleStage {
     }
 
     #[inline(never)]
-    fn register_runnable_task(
-        weighted_tx: TaskInQueue,
+    fn register_runnable_task<T>(
+        weighted_tx: TaskInQueue<T>,
         runnable_queue: &mut TaskQueue,
         sequence_time: &mut usize,
     ) {

From 5b61c5338399086c014a2c8eed26cd2f785a80d4 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 5 Sep 2022 16:36:37 +0900
Subject: [PATCH 2041/3199] save

---
 scheduler/src/lib.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 65cb2b3599db4d..e93fcfd79e7c13 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1219,9 +1219,9 @@ impl ScheduleStage {
     }
 
     #[inline(never)]
-    fn commit_completed_execution<AST: AtScheduleThread>(
+    fn commit_completed_execution<AST: AtScheduleThread, T>(
         ast: AST,
-        ee: &mut ExecutionEnvironment,
+        ee: &mut ExecutionEnvironment<T>,
         address_book: &mut AddressBook,
         commit_time: &mut usize,
         provisioning_tracker_count: &mut usize,

From f90341928363eb41ca8aabcb3700c13d03926a97 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 5 Sep 2022 16:36:58 +0900
Subject: [PATCH 2042/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index e93fcfd79e7c13..2af72eb94178d2 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1203,7 +1203,7 @@ impl ScheduleStage {
         finalized_lock_attempts: Vec<LockAttempt>,
         queue_clock: &usize,
         execute_clock: &mut usize,
-    ) -> Box<ExecutionEnvironment> {
+    ) -> Box<ExecutionEnvironment<T>> {
         let mut rng = rand::thread_rng();
         // load account now from AccountsDb
         task.record_execute_time(*queue_clock, *execute_clock);

From cb02c5e874af4ed3b57e7034a650d1a61c250c20 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 5 Sep 2022 16:37:20 +0900
Subject: [PATCH 2043/3199] save

---
 scheduler/src/lib.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 2af72eb94178d2..e7f0dcbfef4b6f 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1196,10 +1196,10 @@ impl ScheduleStage {
     }
 
     #[inline(never)]
-    fn prepare_scheduled_execution(
+    fn prepare_scheduled_execution<T>(
         address_book: &mut AddressBook,
         unique_weight: UniqueWeight,
-        task: TaskInQueue,
+        task: TaskInQueue<T>,
         finalized_lock_attempts: Vec<LockAttempt>,
         queue_clock: &usize,
         execute_clock: &mut usize,

From bf183b5e2eee46271011219efd7408f17bc32af2 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 5 Sep 2022 16:37:38 +0900
Subject: [PATCH 2044/3199] save

---
 scheduler/src/lib.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index e7f0dcbfef4b6f..80827198ce9669 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1077,10 +1077,10 @@ impl ScheduleStage {
     }
 
     #[inline(never)]
-    fn finalize_lock_for_provisional_execution<AST: AtScheduleThread>(
+    fn finalize_lock_for_provisional_execution<AST: AtScheduleThread, T>(
         ast: AST,
         address_book: &mut AddressBook,
-        next_task: &Task,
+        next_task: &Task<T>,
         tracker: triomphe::Arc<ProvisioningTracker>,
     ) {
         for l in next_task.lock_attempts_mut(ast).iter_mut() {

From 3ff69652ec58a280e29318ba1c85cf31244decbf Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 5 Sep 2022 16:38:12 +0900
Subject: [PATCH 2045/3199] save

---
 scheduler/src/lib.rs | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 80827198ce9669..1d5ecc82b71482 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -899,9 +899,9 @@ impl ScheduleStage {
     }
 
     #[inline(never)]
-    fn pop_from_queue_then_lock<AST: AtScheduleThread>(
+    fn pop_from_queue_then_lock<AST: AtScheduleThread, T>(
         ast: AST,
-        task_sender: &crossbeam_channel::Sender<(TaskInQueue, Vec<LockAttempt>)>,
+        task_sender: &crossbeam_channel::Sender<(TaskInQueue<T>, Vec<LockAttempt>)>,
         runnable_queue: &mut TaskQueue,
         address_book: &mut AddressBook,
         contended_count: &mut usize,
@@ -909,7 +909,7 @@ impl ScheduleStage {
         sequence_clock: &usize,
         queue_clock: &mut usize,
         provisioning_tracker_count: &mut usize,
-    ) -> Option<(UniqueWeight, TaskInQueue, Vec<LockAttempt>)> {
+    ) -> Option<(UniqueWeight, TaskInQueue<T>, Vec<LockAttempt>)> {
         if let Some(mut a) = address_book.fulfilled_provisional_task_ids.pop_last() {
             trace!(
                 "expediate pop from provisional queue [rest: {}]",

From 7139021237ede1ef2e77b894ae16f9af0d67ad03 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 5 Sep 2022 16:38:25 +0900
Subject: [PATCH 2046/3199] save

---
 scheduler/src/lib.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 1d5ecc82b71482..85563a31b823f9 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -838,11 +838,11 @@ impl ScheduleStage {
     }
 
     #[inline(never)]
-    fn select_next_task<'a>(
+    fn select_next_task<'a, T>(
         runnable_queue: &'a mut TaskQueue,
         address_book: &mut AddressBook,
         contended_count: &usize,
-    ) -> Option<(TaskSource, TaskInQueue)> {
+    ) -> Option<(TaskSource, TaskInQueue<T>)> {
         match (
             runnable_queue.heaviest_entry_to_execute(),
             Self::get_heaviest_from_contended(address_book),

From ebb58451ff4a5b8d580dc346763fbf6746c8b751 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 5 Sep 2022 16:38:42 +0900
Subject: [PATCH 2047/3199] save

---
 scheduler/src/lib.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 85563a31b823f9..48b9aef49ea6e9 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -826,14 +826,14 @@ enum TaskSource {
 }
 
 impl ScheduleStage {
-    fn push_to_runnable_queue(task: TaskInQueue, runnable_queue: &mut TaskQueue) {
+    fn push_to_runnable_queue(task: TaskInQueue<T>, runnable_queue: &mut TaskQueue) {
         runnable_queue.add_to_schedule(task.unique_weight, task);
     }
 
     #[inline(never)]
     fn get_heaviest_from_contended<'a>(
         address_book: &'a mut AddressBook,
-    ) -> Option<std::collections::btree_map::OccupiedEntry<'a, UniqueWeight, TaskInQueue>> {
+    ) -> Option<std::collections::btree_map::OccupiedEntry<'a, UniqueWeight, TaskInQueue<T>>> {
         address_book.uncontended_task_ids.last_entry()
     }
 

From 1a505f098437654da36e0fd3f98a9e693281f931 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 5 Sep 2022 16:38:55 +0900
Subject: [PATCH 2048/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 48b9aef49ea6e9..4158c6221ebe9b 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -745,7 +745,7 @@ type TaskQueueOccupiedEntry<'a> =
 
 impl TaskQueue {
     #[inline(never)]
-    fn add_to_schedule(&mut self, unique_weight: UniqueWeight, task: TaskInQueue) {
+    fn add_to_schedule(&mut self, unique_weight: UniqueWeight, task: TaskInQueue<T>) {
         //trace!("TaskQueue::add(): {:?}", unique_weight);
         let pre_existed = self.tasks.insert(unique_weight, task);
         assert!(pre_existed.is_none()); //, "identical shouldn't exist: {:?}", unique_weight);

From d73f20dc4f38119da594e59753beb792149e27c1 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 5 Sep 2022 16:39:08 +0900
Subject: [PATCH 2049/3199] save

---
 scheduler/src/lib.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 4158c6221ebe9b..25e3953b2b89f2 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -701,8 +701,8 @@ impl Task {
 
     #[inline(never)]
     fn index_with_address_book(
-        this: &TaskInQueue,
-        task_sender: &crossbeam_channel::Sender<(TaskInQueue, Vec<LockAttempt>)>,
+        this: &TaskInQueue<T>,
+        task_sender: &crossbeam_channel::Sender<(TaskInQueue<T>, Vec<LockAttempt>)>,
     ) {
         //for lock_attempt in self.lock_attempts_mut(ast).iter() {
         //    lock_attempt.contended_unique_weights().insert_task(unique_weight, Task::clone_in_queue(&self));

From cac9d59a4e6afe7a04673d20c692ed5c5c781b08 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 5 Sep 2022 16:40:08 +0900
Subject: [PATCH 2050/3199] save

---
 scheduler/src/lib.rs | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 25e3953b2b89f2..7277231ceb642a 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -532,12 +532,12 @@ impl LockAttemptsInCell {
 // execute_time ---> exec clock
 // commit_time  -+
 
-impl Task {
+impl<T> Task<T> {
     pub fn new_for_queue<NAST: NotAtScheduleThread>(
         nast: NAST,
         unique_weight: UniqueWeight,
         tx: (SanitizedTransaction, Vec<LockAttempt>),
-    ) -> TaskInQueue {
+    ) -> TaskInQueue<T> {
         TaskInQueue::new(Self {
             for_indexer: LockAttemptsInCell::new(std::cell::RefCell::new(
                 tx.1.iter().map(|a| a.clone_for_test()).collect(),
@@ -557,8 +557,8 @@ impl Task {
     }
 
     #[inline(never)]
-    pub fn clone_in_queue(this: &TaskInQueue) -> TaskInQueue {
-        TaskInQueue::clone(this)
+    pub fn clone_in_queue(this: &TaskInQueue<T>) -> TaskInQueue<T> {
+        TaskInQueue<T>::clone(this)
     }
 
     fn lock_attempts_mut<AST: AtScheduleThread>(

From be46a400f5483435df59b8bd383594de2c2cfa61 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 5 Sep 2022 16:40:38 +0900
Subject: [PATCH 2051/3199] save

---
 scheduler/src/lib.rs | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 7277231ceb642a..f372bf6340b5ca 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -269,13 +269,13 @@ pub struct AddressBook {
 }
 
 #[derive(Debug)]
-struct ProvisioningTracker {
+struct ProvisioningTracker<T> {
     remaining_count: std::sync::atomic::AtomicUsize,
-    task: TaskInQueue,
+    task: TaskInQueue<T>,
 }
 
-impl ProvisioningTracker {
-    fn new(remaining_count: usize, task: TaskInQueue) -> Self {
+impl<T> ProvisioningTracker {
+    fn new(remaining_count: usize, task: TaskInQueue<T>) -> Self {
         Self {
             remaining_count: std::sync::atomic::AtomicUsize::new(remaining_count),
             task,

From 4280657f3631b44fc9aff1aa2646bbc0637fdc84 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 5 Sep 2022 16:40:51 +0900
Subject: [PATCH 2052/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index f372bf6340b5ca..da82d2f433434e 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1081,7 +1081,7 @@ impl ScheduleStage {
         ast: AST,
         address_book: &mut AddressBook,
         next_task: &Task<T>,
-        tracker: triomphe::Arc<ProvisioningTracker>,
+        tracker: triomphe::Arc<ProvisioningTracker<T>>,
     ) {
         for l in next_task.lock_attempts_mut(ast).iter_mut() {
             match l.status {

From a1916993c13b0471132b45aff22bb68f9972091a Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 5 Sep 2022 16:41:04 +0900
Subject: [PATCH 2053/3199] save

---
 scheduler/src/lib.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index da82d2f433434e..2255d2bce95723 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -216,9 +216,9 @@ impl TaskIds {
     }
 
     #[inline(never)]
-    pub fn heaviest_task_cursor(
+    pub fn heaviest_task_cursor<T>(
         &self,
-    ) -> Option<crossbeam_skiplist::map::Entry<'_, UniqueWeight, TaskInQueue>> {
+    ) -> Option<crossbeam_skiplist::map::Entry<'_, UniqueWeight, TaskInQueue<T>>> {
         self.task_ids.back()
     }
 }

From 1f70d8a5e72d6c46d013502cc8d97510fb72bf37 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 5 Sep 2022 16:41:17 +0900
Subject: [PATCH 2054/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 2255d2bce95723..a74f57c7002858 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -200,7 +200,7 @@ pub struct TaskIds {
 
 impl TaskIds {
     #[inline(never)]
-    pub fn insert_task(&self, u: TaskId, task: TaskInQueue) {
+    pub fn insert_task<T>(&self, u: TaskId, task: TaskInQueue<T>) {
         let mut is_inserted = false;
         self.task_ids.get_or_insert_with(u, || {
             is_inserted = true;

From 9c80f7f8b4b1c99701c404173dfb7510a2a25be8 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 5 Sep 2022 16:41:33 +0900
Subject: [PATCH 2055/3199] save

---
 scheduler/src/lib.rs | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index a74f57c7002858..878edd251f8425 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -739,9 +739,9 @@ pub type TaskInQueue<T> = triomphe::Arc<Task<T>>;
 //type TaskQueueOccupiedEntry<'a> = im::hashmap::OccupiedEntry<'a, UniqueWeight, TaskInQueue, std::collections::hash_map::RandomState>;
 //type TaskQueueEntry<'a> = dashmap::mapref::entry::Entry<'a, UniqueWeight, TaskInQueue>;
 //type TaskQueueOccupiedEntry<'a> = dashmap::mapref::entry::OccupiedEntry<'a, UniqueWeight, TaskInQueue, std::collections::hash_map::RandomState>;
-type TaskQueueEntry<'a> = std::collections::btree_map::Entry<'a, UniqueWeight, TaskInQueue>;
-type TaskQueueOccupiedEntry<'a> =
-    std::collections::btree_map::OccupiedEntry<'a, UniqueWeight, TaskInQueue>;
+type TaskQueueEntry<'a, T> = std::collections::btree_map::Entry<'a, UniqueWeight, TaskInQueue<T>>;
+type TaskQueueOccupiedEntry<'a, T> =
+    std::collections::btree_map::OccupiedEntry<'a, UniqueWeight, TaskInQueue<T>>;
 
 impl TaskQueue {
     #[inline(never)]

From 5c0aad6beee87b192055d318a5979f103ca7680d Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 5 Sep 2022 16:42:00 +0900
Subject: [PATCH 2056/3199] save

---
 scheduler/src/lib.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 878edd251f8425..adde55fc87df15 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -743,7 +743,7 @@ type TaskQueueEntry<'a, T> = std::collections::btree_map::Entry<'a, UniqueWeight
 type TaskQueueOccupiedEntry<'a, T> =
     std::collections::btree_map::OccupiedEntry<'a, UniqueWeight, TaskInQueue<T>>;
 
-impl TaskQueue {
+impl<T> TaskQueue<T> {
     #[inline(never)]
     fn add_to_schedule(&mut self, unique_weight: UniqueWeight, task: TaskInQueue<T>) {
         //trace!("TaskQueue::add(): {:?}", unique_weight);
@@ -752,7 +752,7 @@ impl TaskQueue {
     }
 
     #[inline(never)]
-    fn heaviest_entry_to_execute(&mut self) -> Option<TaskQueueOccupiedEntry<'_>> {
+    fn heaviest_entry_to_execute(&mut self) -> Option<TaskQueueOccupiedEntry<'_, T>> {
         self.tasks.last_entry()
     }
 

From fa8d40fc07997cd9539017ec2d7a2765b6abac60 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 5 Sep 2022 16:42:18 +0900
Subject: [PATCH 2057/3199] save

---
 scheduler/src/lib.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index adde55fc87df15..761bb22f1c817c 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -724,8 +724,8 @@ impl<T> Task<T> {
 
 // RunnableQueue, ContendedQueue?
 #[derive(Default, Debug, Clone)]
-pub struct TaskQueue {
-    tasks: std::collections::BTreeMap<UniqueWeight, TaskInQueue>,
+pub struct TaskQueue<T> {
+    tasks: std::collections::BTreeMap<UniqueWeight, TaskInQueue<T>>,
     //tasks: im::OrdMap<UniqueWeight, TaskInQueue>,
     //tasks: im::HashMap<UniqueWeight, TaskInQueue>,
     //tasks: std::sync::Arc<dashmap::DashMap<UniqueWeight, TaskInQueue>>,

From d907ff4e049f183464fa0a5d2a3ad73e8f670541 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 5 Sep 2022 16:42:29 +0900
Subject: [PATCH 2058/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 761bb22f1c817c..69c45ba59c80d5 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1499,7 +1499,7 @@ impl ScheduleStage {
 
     pub fn run<T>(
         max_executing_queue_count: usize,
-        runnable_queue: &mut TaskQueue,
+        runnable_queue: &mut TaskQueue<T>,
         address_book: &mut AddressBook,
         from: &crossbeam_channel::Receiver<SchedulablePayload<T>>,
         to_execute_substage: &crossbeam_channel::Sender<ExecutablePayload<T>>,

From a02466bb611639591d61a50cfad935efe2537ce7 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 5 Sep 2022 16:42:38 +0900
Subject: [PATCH 2059/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 69c45ba59c80d5..534960199126b5 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1300,7 +1300,7 @@ impl ScheduleStage {
     fn _run<AST: AtScheduleThread, T>(
         ast: AST,
         max_executing_queue_count: usize,
-        runnable_queue: &mut TaskQueue,
+        runnable_queue: &mut TaskQueue<T>,
         address_book: &mut AddressBook,
         from_prev: &crossbeam_channel::Receiver<SchedulablePayload<T>>,
         to_execute_substage: &crossbeam_channel::Sender<ExecutablePayload<T>>,

From 83d13c0e23f88ea57c0235d714a00b5eb3344987 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 5 Sep 2022 16:42:49 +0900
Subject: [PATCH 2060/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 534960199126b5..fb1c5617be0945 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1289,7 +1289,7 @@ impl ScheduleStage {
     #[inline(never)]
     fn register_runnable_task<T>(
         weighted_tx: TaskInQueue<T>,
-        runnable_queue: &mut TaskQueue,
+        runnable_queue: &mut TaskQueue<T>,
         sequence_time: &mut usize,
     ) {
         weighted_tx.record_sequence_time(*sequence_time);

From 00585d6c365c1e33470541c09c36f53930b4e7f1 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 5 Sep 2022 16:43:26 +0900
Subject: [PATCH 2061/3199] save

---
 scheduler/src/lib.rs | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index fb1c5617be0945..388f38a206633b 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -826,7 +826,7 @@ enum TaskSource {
 }
 
 impl ScheduleStage {
-    fn push_to_runnable_queue(task: TaskInQueue<T>, runnable_queue: &mut TaskQueue) {
+    fn push_to_runnable_queue(task: TaskInQueue<T>, runnable_queue: &mut TaskQueue<T>) {
         runnable_queue.add_to_schedule(task.unique_weight, task);
     }
 
@@ -902,7 +902,7 @@ impl ScheduleStage {
     fn pop_from_queue_then_lock<AST: AtScheduleThread, T>(
         ast: AST,
         task_sender: &crossbeam_channel::Sender<(TaskInQueue<T>, Vec<LockAttempt>)>,
-        runnable_queue: &mut TaskQueue,
+        runnable_queue: &mut TaskQueue<T>,
         address_book: &mut AddressBook,
         contended_count: &mut usize,
         prefer_immediate: bool,
@@ -1260,7 +1260,7 @@ impl ScheduleStage {
     fn schedule_next_execution<AST: AtScheduleThread, T>(
         ast: AST,
         task_sender: &crossbeam_channel::Sender<(TaskInQueue<T>, Vec<LockAttempt>)>,
-        runnable_queue: &mut TaskQueue,
+        runnable_queue: &mut TaskQueue<T>,
         address_book: &mut AddressBook,
         contended_count: &mut usize,
         prefer_immediate: bool,

From 1692ce5d878fe1015c456fba007799061413bf3e Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 5 Sep 2022 16:43:59 +0900
Subject: [PATCH 2062/3199] save

---
 scheduler/src/lib.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 388f38a206633b..164014b724c9c4 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -274,7 +274,7 @@ struct ProvisioningTracker<T> {
     task: TaskInQueue<T>,
 }
 
-impl<T> ProvisioningTracker {
+impl<T> ProvisioningTracker<T> {
     fn new(remaining_count: usize, task: TaskInQueue<T>) -> Self {
         Self {
             remaining_count: std::sync::atomic::AtomicUsize::new(remaining_count),
@@ -839,7 +839,7 @@ impl ScheduleStage {
 
     #[inline(never)]
     fn select_next_task<'a, T>(
-        runnable_queue: &'a mut TaskQueue,
+        runnable_queue: &'a mut TaskQueue<T>,
         address_book: &mut AddressBook,
         contended_count: &usize,
     ) -> Option<(TaskSource, TaskInQueue<T>)> {

From b51e2e0888472cc287ae8656de5dbeefb72a620f Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 5 Sep 2022 16:44:10 +0900
Subject: [PATCH 2063/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 164014b724c9c4..66de3d581614eb 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -39,7 +39,7 @@ pub struct ExecutionEnvironment<T> {
     pub extra: T,
 }
 
-impl ExecutionEnvironment {
+impl<T> ExecutionEnvironment<T> {
     //fn new(cu: usize) -> Self {
     //    Self {
     //        cu,

From 15c53437fb064cb6ac684832d7a4d76eb252929c Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 5 Sep 2022 16:44:19 +0900
Subject: [PATCH 2064/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 66de3d581614eb..df0502dc0842db 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -265,7 +265,7 @@ pub struct AddressBook {
     book: AddressMap,
     uncontended_task_ids: WeightedTaskIds,
     fulfilled_provisional_task_ids: WeightedTaskIds,
-    stuck_tasks: std::collections::BTreeMap<StuckTaskId, TaskInQueue>,
+    stuck_tasks: std::collections::BTreeMap<StuckTaskId, TaskInQueue<T>>,
 }
 
 #[derive(Debug)]

From e8ec25542a8297f7dc295fd91daa5abf6a6df893 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 5 Sep 2022 16:44:45 +0900
Subject: [PATCH 2065/3199] save

---
 scheduler/src/lib.rs | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index df0502dc0842db..5780236bd6c5e0 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -253,7 +253,7 @@ impl Page {
 //type AddressMap = std::collections::HashMap<Pubkey, PageRc>;
 type AddressMap = std::sync::Arc<dashmap::DashMap<Pubkey, PageRc>>;
 type TaskId = UniqueWeight;
-type WeightedTaskIds = std::collections::BTreeMap<TaskId, TaskInQueue>;
+type WeightedTaskIds<T> = std::collections::BTreeMap<TaskId, TaskInQueue<T>>;
 //type AddressMapEntry<'a, K, V> = std::collections::hash_map::Entry<'a, K, V>;
 type AddressMapEntry<'a> = dashmap::mapref::entry::Entry<'a, Pubkey, PageRc>;
 
@@ -261,10 +261,10 @@ type StuckTaskId = (CU, TaskId);
 
 // needs ttl mechanism and prune
 #[derive(Default)]
-pub struct AddressBook {
+pub struct<T> AddressBook<T> {
     book: AddressMap,
-    uncontended_task_ids: WeightedTaskIds,
-    fulfilled_provisional_task_ids: WeightedTaskIds,
+    uncontended_task_ids: WeightedTaskIds<T>,
+    fulfilled_provisional_task_ids: WeightedTaskIds<T>,
     stuck_tasks: std::collections::BTreeMap<StuckTaskId, TaskInQueue<T>>,
 }
 

From e27903aac7d27da379915b85639c2cb66c7eb332 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 5 Sep 2022 16:44:54 +0900
Subject: [PATCH 2066/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 5780236bd6c5e0..891894bd1b3bba 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -261,7 +261,7 @@ type StuckTaskId = (CU, TaskId);
 
 // needs ttl mechanism and prune
 #[derive(Default)]
-pub struct<T> AddressBook<T> {
+pub struct AddressBook<T> {
     book: AddressMap,
     uncontended_task_ids: WeightedTaskIds<T>,
     fulfilled_provisional_task_ids: WeightedTaskIds<T>,

From e00216fcfe224964f55fca211d04b5d9dd6e3a71 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 5 Sep 2022 16:45:06 +0900
Subject: [PATCH 2067/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 891894bd1b3bba..3477424f8fa978 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1500,7 +1500,7 @@ impl ScheduleStage {
     pub fn run<T>(
         max_executing_queue_count: usize,
         runnable_queue: &mut TaskQueue<T>,
-        address_book: &mut AddressBook,
+        address_book: &mut AddressBook<T>,
         from: &crossbeam_channel::Receiver<SchedulablePayload<T>>,
         to_execute_substage: &crossbeam_channel::Sender<ExecutablePayload<T>>,
         from_execute_substage: &crossbeam_channel::Receiver<UnlockablePayload<T>>,

From 3fb5ec9d4a13cdbff41274fc8aee5abb53fcb169 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 5 Sep 2022 16:45:24 +0900
Subject: [PATCH 2068/3199] save

---
 scheduler/src/lib.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 3477424f8fa978..6b14c61f1fa615 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1261,7 +1261,7 @@ impl ScheduleStage {
         ast: AST,
         task_sender: &crossbeam_channel::Sender<(TaskInQueue<T>, Vec<LockAttempt>)>,
         runnable_queue: &mut TaskQueue<T>,
-        address_book: &mut AddressBook,
+        address_book: &mut AddressBook<T>,
         contended_count: &mut usize,
         prefer_immediate: bool,
         sequence_time: &usize,
@@ -1301,7 +1301,7 @@ impl ScheduleStage {
         ast: AST,
         max_executing_queue_count: usize,
         runnable_queue: &mut TaskQueue<T>,
-        address_book: &mut AddressBook,
+        address_book: &mut AddressBook<T>,
         from_prev: &crossbeam_channel::Receiver<SchedulablePayload<T>>,
         to_execute_substage: &crossbeam_channel::Sender<ExecutablePayload<T>>,
         from_exec: &crossbeam_channel::Receiver<UnlockablePayload<T>>,

From 37cf22a35f7ad8db13d957f3dbcf354a731caf6b Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 5 Sep 2022 16:45:39 +0900
Subject: [PATCH 2069/3199] save

---
 scheduler/src/lib.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 6b14c61f1fa615..894dbd90eeb297 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1197,7 +1197,7 @@ impl ScheduleStage {
 
     #[inline(never)]
     fn prepare_scheduled_execution<T>(
-        address_book: &mut AddressBook,
+        address_book: &mut AddressBook<T>,
         unique_weight: UniqueWeight,
         task: TaskInQueue<T>,
         finalized_lock_attempts: Vec<LockAttempt>,
@@ -1222,7 +1222,7 @@ impl ScheduleStage {
     fn commit_completed_execution<AST: AtScheduleThread, T>(
         ast: AST,
         ee: &mut ExecutionEnvironment<T>,
-        address_book: &mut AddressBook,
+        address_book: &mut AddressBook<T>,
         commit_time: &mut usize,
         provisioning_tracker_count: &mut usize,
     ) {

From 69696609b1056cf65104628cefc69908c0c4d66d Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 5 Sep 2022 16:46:01 +0900
Subject: [PATCH 2070/3199] save

---
 scheduler/src/lib.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 894dbd90eeb297..88be0db6feb892 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1105,7 +1105,7 @@ impl ScheduleStage {
     #[inline(never)]
     fn reset_lock_for_failed_execution<AST: AtScheduleThread>(
         ast: AST,
-        address_book: &mut AddressBook,
+        address_book: &mut AddressBook<T>,
         unique_weight: &UniqueWeight,
         lock_attempts: &mut [LockAttempt],
     ) {
@@ -1117,7 +1117,7 @@ impl ScheduleStage {
     #[inline(never)]
     fn unlock_after_execution<AST: AtScheduleThread>(
         ast: AST,
-        address_book: &mut AddressBook,
+        address_book: &mut AddressBook<T>,
         lock_attempts: &mut [LockAttempt],
         provisioning_tracker_count: &mut usize,
         cu: CU,

From 5bf8809e1b5f4ac2ce4994b4d695ebe4fbcf5798 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 5 Sep 2022 16:46:12 +0900
Subject: [PATCH 2071/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 88be0db6feb892..e1e77578bc04e3 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1079,7 +1079,7 @@ impl ScheduleStage {
     #[inline(never)]
     fn finalize_lock_for_provisional_execution<AST: AtScheduleThread, T>(
         ast: AST,
-        address_book: &mut AddressBook,
+        address_book: &mut AddressBook<T>,
         next_task: &Task<T>,
         tracker: triomphe::Arc<ProvisioningTracker<T>>,
     ) {

From 45b1e4a9701c90661b3aae5f95ec041e2a6deff4 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 5 Sep 2022 16:46:22 +0900
Subject: [PATCH 2072/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index e1e77578bc04e3..21af26b1471a0a 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -903,7 +903,7 @@ impl ScheduleStage {
         ast: AST,
         task_sender: &crossbeam_channel::Sender<(TaskInQueue<T>, Vec<LockAttempt>)>,
         runnable_queue: &mut TaskQueue<T>,
-        address_book: &mut AddressBook,
+        address_book: &mut AddressBook<T>,
         contended_count: &mut usize,
         prefer_immediate: bool,
         sequence_clock: &usize,

From 0fb2b210f7e0169c5f76371cd5ab474581b32ebb Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 5 Sep 2022 16:46:36 +0900
Subject: [PATCH 2073/3199] save

---
 scheduler/src/lib.rs | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 21af26b1471a0a..92f1de94065bfc 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -831,8 +831,8 @@ impl ScheduleStage {
     }
 
     #[inline(never)]
-    fn get_heaviest_from_contended<'a>(
-        address_book: &'a mut AddressBook,
+    fn get_heaviest_from_contended<'a, T>(
+        address_book: &'a mut AddressBook<T>,
     ) -> Option<std::collections::btree_map::OccupiedEntry<'a, UniqueWeight, TaskInQueue<T>>> {
         address_book.uncontended_task_ids.last_entry()
     }
@@ -840,7 +840,7 @@ impl ScheduleStage {
     #[inline(never)]
     fn select_next_task<'a, T>(
         runnable_queue: &'a mut TaskQueue<T>,
-        address_book: &mut AddressBook,
+        address_book: &mut AddressBook<T>,
         contended_count: &usize,
     ) -> Option<(TaskSource, TaskInQueue<T>)> {
         match (

From 10f0f4c157b270f25da867c29acbbdd51ab0a338 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 5 Sep 2022 16:46:58 +0900
Subject: [PATCH 2074/3199] save

---
 scheduler/src/lib.rs | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 92f1de94065bfc..c2ad5fe992c84d 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -762,11 +762,11 @@ impl<T> TaskQueue<T> {
 }
 
 #[inline(never)]
-fn attempt_lock_for_execution<'a, AST: AtScheduleThread>(
+fn attempt_lock_for_execution<'a, AST: AtScheduleThread, T>(
     ast: AST,
     from_runnable: bool,
     prefer_immediate: bool,
-    address_book: &mut AddressBook,
+    address_book: &mut AddressBook<T>,
     unique_weight: &UniqueWeight,
     message_hash: &'a Hash,
     lock_attempts: &mut [LockAttempt],
@@ -777,7 +777,7 @@ fn attempt_lock_for_execution<'a, AST: AtScheduleThread>(
     let mut busiest_page_cu = 1;
 
     for attempt in lock_attempts.iter_mut() {
-        let cu = AddressBook::attempt_lock_address(
+        let cu = AddressBook::<T>::attempt_lock_address(
             ast,
             from_runnable,
             prefer_immediate,

From 136bdf2bba8f58a0ccb68754415bf4ce9065b170 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 5 Sep 2022 16:47:11 +0900
Subject: [PATCH 2075/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index c2ad5fe992c84d..0e9e5904ede361 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -301,7 +301,7 @@ impl<T> ProvisioningTracker<T> {
     }
 }
 
-impl AddressBook {
+impl<T> AddressBook<T> {
     #[inline(never)]
     fn attempt_lock_address<AST: AtScheduleThread>(
         ast: AST,

From 00941f8236d00a8a3fb8d2c436f204f39e59d1ff Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 5 Sep 2022 16:47:24 +0900
Subject: [PATCH 2076/3199] save

---
 scheduler/src/lib.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 0e9e5904ede361..7b82dae1412a27 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -224,10 +224,10 @@ impl TaskIds {
 }
 
 #[derive(Debug)]
-pub struct Page {
+pub struct Page<T> {
     current_usage: Usage,
     next_usage: Usage,
-    provisional_task_ids: Vec<triomphe::Arc<ProvisioningTracker>>,
+    provisional_task_ids: Vec<triomphe::Arc<ProvisioningTracker<T>>>,
     cu: CU,
     //loaded account from Accounts db
     //comulative_cu for qos; i.e. track serialized cumulative keyed by addresses and bail out block

From fdb1b2ad6068a03f33632565e6e5071274ed96b2 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 5 Sep 2022 16:47:44 +0900
Subject: [PATCH 2077/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 7b82dae1412a27..77663bc119e17d 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -114,7 +114,7 @@ unsafe trait AtScheduleThread: Copy {}
 pub unsafe trait NotAtScheduleThread: Copy {}
 
 impl PageRc {
-    fn page_mut<AST: AtScheduleThread>(&self, _ast: AST) -> std::cell::RefMut<'_, Page> {
+    fn page_mut<AST: AtScheduleThread, T>(&self, _ast: AST) -> std::cell::RefMut<'_, Page, T> {
         self.0 .0.borrow_mut()
     }
 }

From 32cf75a7397f0ebd511d313e3aa2af4bde16fa5f Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 5 Sep 2022 16:48:40 +0900
Subject: [PATCH 2078/3199] save

---
 scheduler/src/lib.rs | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 77663bc119e17d..fc8a3d00ba6ffb 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -33,7 +33,7 @@ pub struct ExecutionEnvironment<T> {
     //accounts: Vec<i8>,
     pub cu: CU,
     pub unique_weight: UniqueWeight,
-    pub task: TaskInQueue,
+    pub task: TaskInQueue<T>,
     pub finalized_lock_attempts: Vec<LockAttempt>,
     pub is_reindexed: bool,
     pub extra: T,
@@ -195,7 +195,7 @@ pub enum RequestedUsage {
 
 #[derive(Debug, Default)]
 pub struct TaskIds {
-    task_ids: crossbeam_skiplist::SkipMap<UniqueWeight, TaskInQueue>,
+    task_ids: crossbeam_skiplist::SkipMap<UniqueWeight, TaskInQueue<T>>,
 }
 
 impl TaskIds {
@@ -1103,7 +1103,7 @@ impl ScheduleStage {
     }
 
     #[inline(never)]
-    fn reset_lock_for_failed_execution<AST: AtScheduleThread>(
+    fn reset_lock_for_failed_execution<AST: AtScheduleThread, T>(
         ast: AST,
         address_book: &mut AddressBook<T>,
         unique_weight: &UniqueWeight,
@@ -1115,7 +1115,7 @@ impl ScheduleStage {
     }
 
     #[inline(never)]
-    fn unlock_after_execution<AST: AtScheduleThread>(
+    fn unlock_after_execution<AST: AtScheduleThread, T>(
         ast: AST,
         address_book: &mut AddressBook<T>,
         lock_attempts: &mut [LockAttempt],

From 79c50c3e0a3a1735fab4be34acdf0534d117e71c Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 5 Sep 2022 16:49:45 +0900
Subject: [PATCH 2079/3199] save

---
 scheduler/src/lib.rs | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index fc8a3d00ba6ffb..946fe0c3f00bda 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -127,12 +127,12 @@ enum LockStatus {
 }
 
 #[derive(Debug)]
-pub struct LockAttempt {
+pub struct LockAttempt<T> {
     target: PageRc,
     status: LockStatus,
     requested_usage: RequestedUsage,
     //pub heaviest_uncontended: arc_swap::ArcSwapOption<Task>,
-    pub heaviest_uncontended: Option<TaskInQueue>,
+    pub heaviest_uncontended: Option<TaskInQueue<T>>,
     //remembered: bool,
 }
 
@@ -194,11 +194,11 @@ pub enum RequestedUsage {
 }
 
 #[derive(Debug, Default)]
-pub struct TaskIds {
+pub struct TaskIds<T> {
     task_ids: crossbeam_skiplist::SkipMap<UniqueWeight, TaskInQueue<T>>,
 }
 
-impl TaskIds {
+impl<T> TaskIds<T> {
     #[inline(never)]
     pub fn insert_task<T>(&self, u: TaskId, task: TaskInQueue<T>) {
         let mut is_inserted = false;
@@ -558,7 +558,7 @@ impl<T> Task<T> {
 
     #[inline(never)]
     pub fn clone_in_queue(this: &TaskInQueue<T>) -> TaskInQueue<T> {
-        TaskInQueue<T>::clone(this)
+        TaskInQueue::<T>::clone(this)
     }
 
     fn lock_attempts_mut<AST: AtScheduleThread>(
@@ -826,7 +826,7 @@ enum TaskSource {
 }
 
 impl ScheduleStage {
-    fn push_to_runnable_queue(task: TaskInQueue<T>, runnable_queue: &mut TaskQueue<T>) {
+    fn push_to_runnable_queue<T>(task: TaskInQueue<T>, runnable_queue: &mut TaskQueue<T>) {
         runnable_queue.add_to_schedule(task.unique_weight, task);
     }
 

From 494730ea2e7789b435971f88e81b222e0ef0efd2 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 5 Sep 2022 16:51:21 +0900
Subject: [PATCH 2080/3199] save

---
 scheduler/src/lib.rs | 44 ++++++++++++++++++++++----------------------
 1 file changed, 22 insertions(+), 22 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 946fe0c3f00bda..6b1f1d9facb82a 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -34,7 +34,7 @@ pub struct ExecutionEnvironment<T> {
     pub cu: CU,
     pub unique_weight: UniqueWeight,
     pub task: TaskInQueue<T>,
-    pub finalized_lock_attempts: Vec<LockAttempt>,
+    pub finalized_lock_attempts: Vec<LockAttempt<T>>,
     pub is_reindexed: bool,
     pub extra: T,
 }
@@ -136,7 +136,7 @@ pub struct LockAttempt<T> {
     //remembered: bool,
 }
 
-impl LockAttempt {
+impl<T> LockAttempt<T> {
     pub fn new(target: PageRc, requested_usage: RequestedUsage) -> Self {
         Self {
             target,
@@ -308,7 +308,7 @@ impl<T> AddressBook<T> {
         from_runnable: bool,
         prefer_immediate: bool,
         unique_weight: &UniqueWeight,
-        attempt: &mut LockAttempt,
+        attempt: &mut LockAttempt<T>,
     ) -> CU {
         let LockAttempt {
             target,
@@ -379,7 +379,7 @@ impl<T> AddressBook<T> {
     fn reset_lock<AST: AtScheduleThread>(
         &mut self,
         ast: AST,
-        attempt: &mut LockAttempt,
+        attempt: &mut LockAttempt<T>,
         after_execution: bool,
     ) -> bool {
         match attempt.status {
@@ -399,7 +399,7 @@ impl<T> AddressBook<T> {
     }
 
     #[inline(never)]
-    fn unlock<AST: AtScheduleThread>(&mut self, ast: AST, attempt: &mut LockAttempt) -> bool {
+    fn unlock<AST: AtScheduleThread>(&mut self, ast: AST, attempt: &mut LockAttempt<T>) -> bool {
         //debug_assert!(attempt.is_success());
 
         let mut newly_uncontended = false;
@@ -434,7 +434,7 @@ impl<T> AddressBook<T> {
     }
 
     #[inline(never)]
-    fn cancel<AST: AtScheduleThread>(&mut self, ast: AST, attempt: &mut LockAttempt) {
+    fn cancel<AST: AtScheduleThread>(&mut self, ast: AST, attempt: &mut LockAttempt<T>) {
         let mut page = attempt.target.page_mut(ast);
 
         match page.next_usage {
@@ -516,13 +516,13 @@ pub struct Task<T> {
 }
 
 #[derive(Debug)]
-pub struct LockAttemptsInCell(std::cell::RefCell<Vec<LockAttempt>>);
+pub struct LockAttemptsInCell<T>(std::cell::RefCell<Vec<LockAttempt<T>>>);
 
 unsafe impl Send for LockAttemptsInCell {}
 unsafe impl Sync for LockAttemptsInCell {}
 
 impl LockAttemptsInCell {
-    fn new(ll: std::cell::RefCell<Vec<LockAttempt>>) -> Self {
+    fn new(ll: std::cell::RefCell<Vec<LockAttempt<T>>>) -> Self {
         Self(ll)
     }
 }
@@ -536,7 +536,7 @@ impl<T> Task<T> {
     pub fn new_for_queue<NAST: NotAtScheduleThread>(
         nast: NAST,
         unique_weight: UniqueWeight,
-        tx: (SanitizedTransaction, Vec<LockAttempt>),
+        tx: (SanitizedTransaction, Vec<LockAttempt<T>>),
     ) -> TaskInQueue<T> {
         TaskInQueue::new(Self {
             for_indexer: LockAttemptsInCell::new(std::cell::RefCell::new(
@@ -564,14 +564,14 @@ impl<T> Task<T> {
     fn lock_attempts_mut<AST: AtScheduleThread>(
         &self,
         _ast: AST,
-    ) -> std::cell::RefMut<'_, Vec<LockAttempt>> {
+    ) -> std::cell::RefMut<'_, Vec<LockAttempt<T>>> {
         self.tx.1 .0.borrow_mut()
     }
 
     fn lock_attempts_not_mut<NAST: NotAtScheduleThread>(
         &self,
         _nast: NAST,
-    ) -> std::cell::Ref<'_, Vec<LockAttempt>> {
+    ) -> std::cell::Ref<'_, Vec<LockAttempt<T>>> {
         self.tx.1 .0.borrow()
     }
 
@@ -702,7 +702,7 @@ impl<T> Task<T> {
     #[inline(never)]
     fn index_with_address_book(
         this: &TaskInQueue<T>,
-        task_sender: &crossbeam_channel::Sender<(TaskInQueue<T>, Vec<LockAttempt>)>,
+        task_sender: &crossbeam_channel::Sender<(TaskInQueue<T>, Vec<LockAttempt<T>>)>,
     ) {
         //for lock_attempt in self.lock_attempts_mut(ast).iter() {
         //    lock_attempt.contended_unique_weights().insert_task(unique_weight, Task::clone_in_queue(&self));
@@ -769,7 +769,7 @@ fn attempt_lock_for_execution<'a, AST: AtScheduleThread, T>(
     address_book: &mut AddressBook<T>,
     unique_weight: &UniqueWeight,
     message_hash: &'a Hash,
-    lock_attempts: &mut [LockAttempt],
+    lock_attempts: &mut [LockAttempt<T>],
 ) -> (usize, usize, CU) {
     // no short-cuircuit; we at least all need to add to the contended queue
     let mut unlockable_count = 0;
@@ -800,7 +800,7 @@ fn attempt_lock_for_execution<'a, AST: AtScheduleThread, T>(
     (unlockable_count, provisional_count, busiest_page_cu)
 }
 
-type PreprocessedTransaction = (SanitizedTransaction, Vec<LockAttempt>);
+type PreprocessedTransaction<T> = (SanitizedTransaction, Vec<LockAttempt<T>>);
 
 /*
 pub fn get_transaction_priority_details(tx: &SanitizedTransaction) -> u64 {
@@ -901,7 +901,7 @@ impl ScheduleStage {
     #[inline(never)]
     fn pop_from_queue_then_lock<AST: AtScheduleThread, T>(
         ast: AST,
-        task_sender: &crossbeam_channel::Sender<(TaskInQueue<T>, Vec<LockAttempt>)>,
+        task_sender: &crossbeam_channel::Sender<(TaskInQueue<T>, Vec<LockAttempt<T>>)>,
         runnable_queue: &mut TaskQueue<T>,
         address_book: &mut AddressBook<T>,
         contended_count: &mut usize,
@@ -909,7 +909,7 @@ impl ScheduleStage {
         sequence_clock: &usize,
         queue_clock: &mut usize,
         provisioning_tracker_count: &mut usize,
-    ) -> Option<(UniqueWeight, TaskInQueue<T>, Vec<LockAttempt>)> {
+    ) -> Option<(UniqueWeight, TaskInQueue<T>, Vec<LockAttempt<T>>)> {
         if let Some(mut a) = address_book.fulfilled_provisional_task_ids.pop_last() {
             trace!(
                 "expediate pop from provisional queue [rest: {}]",
@@ -1107,7 +1107,7 @@ impl ScheduleStage {
         ast: AST,
         address_book: &mut AddressBook<T>,
         unique_weight: &UniqueWeight,
-        lock_attempts: &mut [LockAttempt],
+        lock_attempts: &mut [LockAttempt<T>],
     ) {
         for l in lock_attempts {
             address_book.reset_lock(ast, l, false);
@@ -1118,7 +1118,7 @@ impl ScheduleStage {
     fn unlock_after_execution<AST: AtScheduleThread, T>(
         ast: AST,
         address_book: &mut AddressBook<T>,
-        lock_attempts: &mut [LockAttempt],
+        lock_attempts: &mut [LockAttempt<T>],
         provisioning_tracker_count: &mut usize,
         cu: CU,
     ) {
@@ -1191,7 +1191,7 @@ impl ScheduleStage {
                 }
             }
 
-            // todo: mem::forget and panic in LockAttempt::drop()
+            // todo: mem::forget and panic in LockAttempt<T>::drop()
         }
     }
 
@@ -1200,7 +1200,7 @@ impl ScheduleStage {
         address_book: &mut AddressBook<T>,
         unique_weight: UniqueWeight,
         task: TaskInQueue<T>,
-        finalized_lock_attempts: Vec<LockAttempt>,
+        finalized_lock_attempts: Vec<LockAttempt<T>>,
         queue_clock: &usize,
         execute_clock: &mut usize,
     ) -> Box<ExecutionEnvironment<T>> {
@@ -1259,7 +1259,7 @@ impl ScheduleStage {
     #[inline(never)]
     fn schedule_next_execution<AST: AtScheduleThread, T>(
         ast: AST,
-        task_sender: &crossbeam_channel::Sender<(TaskInQueue<T>, Vec<LockAttempt>)>,
+        task_sender: &crossbeam_channel::Sender<(TaskInQueue<T>, Vec<LockAttempt<T>>)>,
         runnable_queue: &mut TaskQueue<T>,
         address_book: &mut AddressBook<T>,
         contended_count: &mut usize,
@@ -1347,7 +1347,7 @@ impl ScheduleStage {
             (&ee_sender, Some(h))
         };
         let (task_sender, task_receiver) =
-            crossbeam_channel::unbounded::<(TaskInQueue, Vec<LockAttempt>)>();
+            crossbeam_channel::unbounded::<(TaskInQueue, Vec<LockAttempt<T>>)>();
         let indexer_count = std::env::var("INDEXER_COUNT")
             .unwrap_or(format!("{}", 4))
             .parse::<usize>()

From dfe8e0e2bb691a40ed9e8d5a95721f56f0c21b73 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 5 Sep 2022 16:51:37 +0900
Subject: [PATCH 2081/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 6b1f1d9facb82a..8878a63da01847 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -157,7 +157,7 @@ impl<T> LockAttempt<T> {
         }
     }
 
-    pub fn contended_unique_weights(&self) -> &TaskIds {
+    pub fn contended_unique_weights<T>(&self) -> &TaskIds<T> {
         &self.target.0 .1
     }
 }

From 5958c83c82250e3929ceea4bff8603c2143acc93 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 5 Sep 2022 16:52:07 +0900
Subject: [PATCH 2082/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 8878a63da01847..7d251becdec043 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -114,7 +114,7 @@ unsafe trait AtScheduleThread: Copy {}
 pub unsafe trait NotAtScheduleThread: Copy {}
 
 impl PageRc {
-    fn page_mut<AST: AtScheduleThread, T>(&self, _ast: AST) -> std::cell::RefMut<'_, Page, T> {
+    fn page_mut<AST: AtScheduleThread, T>(&self, _ast: AST) -> std::cell::RefMut<'_, Page<T>> {
         self.0 .0.borrow_mut()
     }
 }

From b77ad323aa35502179cd623cdb20baea9faf8e42 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 5 Sep 2022 16:52:23 +0900
Subject: [PATCH 2083/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 7d251becdec043..573e4ceba46b3e 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -521,7 +521,7 @@ pub struct LockAttemptsInCell<T>(std::cell::RefCell<Vec<LockAttempt<T>>>);
 unsafe impl Send for LockAttemptsInCell {}
 unsafe impl Sync for LockAttemptsInCell {}
 
-impl LockAttemptsInCell {
+impl<T> LockAttemptsInCell<T> {
     fn new(ll: std::cell::RefCell<Vec<LockAttempt<T>>>) -> Self {
         Self(ll)
     }

From 848d4dee56f79d170bcdc0223d3f49c5f6b23b84 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 5 Sep 2022 16:52:38 +0900
Subject: [PATCH 2084/3199] save

---
 scheduler/src/lib.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 573e4ceba46b3e..5aa728d93d03d3 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -518,8 +518,8 @@ pub struct Task<T> {
 #[derive(Debug)]
 pub struct LockAttemptsInCell<T>(std::cell::RefCell<Vec<LockAttempt<T>>>);
 
-unsafe impl Send for LockAttemptsInCell {}
-unsafe impl Sync for LockAttemptsInCell {}
+unsafe impl<T> Send for LockAttemptsInCell<T> {}
+unsafe impl<T> Sync for LockAttemptsInCell<T> {}
 
 impl<T> LockAttemptsInCell<T> {
     fn new(ll: std::cell::RefCell<Vec<LockAttempt<T>>>) -> Self {

From c4ba271dabbc6c94cf94f38f6038c996c16fcbe2 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 5 Sep 2022 16:52:50 +0900
Subject: [PATCH 2085/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 5aa728d93d03d3..d169465c3eed2e 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -234,7 +234,7 @@ pub struct Page<T> {
     //producing as soon as any one of cu from the executing thread reaches to the limit
 }
 
-impl Page {
+impl<T> Page<T> {
     fn new(current_usage: Usage) -> Self {
         Self {
             current_usage,

From b42b7c520c674c337149d71467a02fbc113b02a7 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 5 Sep 2022 16:53:03 +0900
Subject: [PATCH 2086/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index d169465c3eed2e..d66e806f2784fd 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -511,7 +511,7 @@ pub struct Task<T> {
     pub queue_end_time: std::sync::atomic::AtomicUsize,
     pub execute_time: std::sync::atomic::AtomicUsize,
     pub commit_time: std::sync::atomic::AtomicUsize,
-    pub for_indexer: LockAttemptsInCell,
+    pub for_indexer: LockAttemptsInCell<T>,
     pub extra: T,
 }
 

From 875aa10bf9f494016b9840e386046536a373bb81 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 5 Sep 2022 16:53:11 +0900
Subject: [PATCH 2087/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index d66e806f2784fd..d3778b92b5cf71 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -501,7 +501,7 @@ struct Bundle {
 #[derive(Debug)]
 pub struct Task<T> {
     unique_weight: UniqueWeight,
-    pub tx: (SanitizedTransaction, LockAttemptsInCell), // actually should be Bundle
+    pub tx: (SanitizedTransaction, LockAttemptsInCell<T>), // actually should be Bundle
     pub contention_count: std::sync::atomic::AtomicUsize,
     pub busiest_page_cu: std::sync::atomic::AtomicU64,
     pub uncontended: std::sync::atomic::AtomicUsize,

From 29574287fe4748ffe20c3cb6e0b8d43bc5975771 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 5 Sep 2022 16:53:29 +0900
Subject: [PATCH 2088/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index d3778b92b5cf71..c6a49c7b1f1739 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -19,7 +19,7 @@ type PageRcInner<T> = std::rc::Rc<T>;
 unsafe impl Send for PageRc {}
 */
 
-type PageRcInner = triomphe::Arc<(std::cell::RefCell<Page>, TaskIds)>;
+type PageRcInner<T> = triomphe::Arc<(std::cell::RefCell<Page<T>>, TaskIds<T>)>;
 
 #[derive(Debug, Clone)]
 pub struct PageRc(PageRcInner);

From 481995a46ea49810f6c32622fa1c7501c2280118 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 5 Sep 2022 16:53:40 +0900
Subject: [PATCH 2089/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index c6a49c7b1f1739..0a219ba8388f65 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -22,7 +22,7 @@ unsafe impl Send for PageRc {}
 type PageRcInner<T> = triomphe::Arc<(std::cell::RefCell<Page<T>>, TaskIds<T>)>;
 
 #[derive(Debug, Clone)]
-pub struct PageRc(PageRcInner);
+pub struct PageRc<T>(PageRcInner<T>);
 unsafe impl Send for PageRc {}
 unsafe impl Sync for PageRc {}
 

From db1a421489ad5d3838a4f7b089e05a7ea305abf4 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 5 Sep 2022 16:53:53 +0900
Subject: [PATCH 2090/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 0a219ba8388f65..fe43d425a7ebd8 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -462,7 +462,7 @@ pub struct Preloader {
 
 impl Preloader {
     #[inline(never)]
-    pub fn load(&self, address: Pubkey) -> PageRc {
+    pub fn load<T>(&self, address: Pubkey) -> PageRc<T> {
         PageRc::clone(&self.book.entry(address).or_insert_with(|| {
             PageRc(PageRcInner::new((
                 core::cell::RefCell::new(Page::new(Usage::unused())),

From 2af135c8821ac727f513d750b72264c830425047 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 5 Sep 2022 16:54:54 +0900
Subject: [PATCH 2091/3199] save

---
 scheduler/src/lib.rs | 18 +++++++++---------
 1 file changed, 9 insertions(+), 9 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index fe43d425a7ebd8..6f33ce9c0d3bc4 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -23,8 +23,8 @@ type PageRcInner<T> = triomphe::Arc<(std::cell::RefCell<Page<T>>, TaskIds<T>)>;
 
 #[derive(Debug, Clone)]
 pub struct PageRc<T>(PageRcInner<T>);
-unsafe impl Send for PageRc {}
-unsafe impl Sync for PageRc {}
+unsafe impl<T> Send for PageRc<T> {}
+unsafe impl<T> Sync for PageRc<T> {}
 
 type CU = u64;
 
@@ -113,8 +113,8 @@ impl<T> ExecutionEnvironment<T> {
 unsafe trait AtScheduleThread: Copy {}
 pub unsafe trait NotAtScheduleThread: Copy {}
 
-impl PageRc {
-    fn page_mut<AST: AtScheduleThread, T>(&self, _ast: AST) -> std::cell::RefMut<'_, Page<T>> {
+impl<T> PageRc<T> {
+    fn page_mut<AST: AtScheduleThread>(&self, _ast: AST) -> std::cell::RefMut<'_, Page<T>> {
         self.0 .0.borrow_mut()
     }
 }
@@ -128,7 +128,7 @@ enum LockStatus {
 
 #[derive(Debug)]
 pub struct LockAttempt<T> {
-    target: PageRc,
+    target: PageRc<T>,
     status: LockStatus,
     requested_usage: RequestedUsage,
     //pub heaviest_uncontended: arc_swap::ArcSwapOption<Task>,
@@ -137,7 +137,7 @@ pub struct LockAttempt<T> {
 }
 
 impl<T> LockAttempt<T> {
-    pub fn new(target: PageRc, requested_usage: RequestedUsage) -> Self {
+    pub fn new(target: PageRc<T>, requested_usage: RequestedUsage) -> Self {
         Self {
             target,
             status: LockStatus::Succeded,
@@ -250,12 +250,12 @@ impl<T> Page<T> {
     }
 }
 
-//type AddressMap = std::collections::HashMap<Pubkey, PageRc>;
-type AddressMap = std::sync::Arc<dashmap::DashMap<Pubkey, PageRc>>;
+//type AddressMap = std::collections::HashMap<Pubkey, PageRc<T>>;
+type AddressMap<T> = std::sync::Arc<dashmap::DashMap<Pubkey, PageRc<T>>>;
 type TaskId = UniqueWeight;
 type WeightedTaskIds<T> = std::collections::BTreeMap<TaskId, TaskInQueue<T>>;
 //type AddressMapEntry<'a, K, V> = std::collections::hash_map::Entry<'a, K, V>;
-type AddressMapEntry<'a> = dashmap::mapref::entry::Entry<'a, Pubkey, PageRc>;
+type AddressMapEntry<'a, T> = dashmap::mapref::entry::Entry<'a, Pubkey, PageRc<T>>;
 
 type StuckTaskId = (CU, TaskId);
 

From 42570462e84ef726a9356740626b1356258191ad Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 5 Sep 2022 16:55:12 +0900
Subject: [PATCH 2092/3199] save

---
 scheduler/src/lib.rs | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 6f33ce9c0d3bc4..5e38032561bb48 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -456,13 +456,13 @@ impl<T> AddressBook<T> {
 }
 
 #[derive(Debug)]
-pub struct Preloader {
-    book: AddressMap,
+pub struct Preloader<T> {
+    book: AddressMap<T>,
 }
 
-impl Preloader {
+impl<T> Preloader<T> {
     #[inline(never)]
-    pub fn load<T>(&self, address: Pubkey) -> PageRc<T> {
+    pub fn load(&self, address: Pubkey) -> PageRc<T> {
         PageRc::clone(&self.book.entry(address).or_insert_with(|| {
             PageRc(PageRcInner::new((
                 core::cell::RefCell::new(Page::new(Usage::unused())),

From 453e3482b579d6c1e74ef4138266e42af4484409 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 5 Sep 2022 16:55:34 +0900
Subject: [PATCH 2093/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 5e38032561bb48..65409a2dede704 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -448,7 +448,7 @@ impl<T> AddressBook<T> {
         }
     }
 
-    pub fn preloader(&self) -> Preloader {
+    pub fn preloader(&self) -> Preloader<T> {
         Preloader {
             book: std::sync::Arc::clone(&self.book),
         }

From abfa385d6a7e230d7e341a3c6f1bbf4b6ed2765b Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 5 Sep 2022 16:55:43 +0900
Subject: [PATCH 2094/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 65409a2dede704..4795649eedbf25 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -262,7 +262,7 @@ type StuckTaskId = (CU, TaskId);
 // needs ttl mechanism and prune
 #[derive(Default)]
 pub struct AddressBook<T> {
-    book: AddressMap,
+    book: AddressMap<T>,
     uncontended_task_ids: WeightedTaskIds<T>,
     fulfilled_provisional_task_ids: WeightedTaskIds<T>,
     stuck_tasks: std::collections::BTreeMap<StuckTaskId, TaskInQueue<T>>,

From fee590e021b92fc7c09a1e45c4410ec049c433f5 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 5 Sep 2022 21:29:41 +0900
Subject: [PATCH 2095/3199] save

---
 scheduler/src/lib.rs | 278 +++++++++++++++++++++----------------------
 1 file changed, 138 insertions(+), 140 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 4795649eedbf25..b5a1d91d31a84c 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -19,27 +19,26 @@ type PageRcInner<T> = std::rc::Rc<T>;
 unsafe impl Send for PageRc {}
 */
 
-type PageRcInner<T> = triomphe::Arc<(std::cell::RefCell<Page<T>>, TaskIds<T>)>;
+type PageRcInner = triomphe::Arc<(std::cell::RefCell<Page>, TaskIds)>;
 
 #[derive(Debug, Clone)]
-pub struct PageRc<T>(PageRcInner<T>);
-unsafe impl<T> Send for PageRc<T> {}
-unsafe impl<T> Sync for PageRc<T> {}
+pub struct PageRc(PageRcInner);
+unsafe impl Send for PageRc {}
+unsafe impl Sync for PageRc {}
 
 type CU = u64;
 
 #[derive(Debug)]
-pub struct ExecutionEnvironment<T> {
+pub struct ExecutionEnvironment {
     //accounts: Vec<i8>,
     pub cu: CU,
     pub unique_weight: UniqueWeight,
-    pub task: TaskInQueue<T>,
-    pub finalized_lock_attempts: Vec<LockAttempt<T>>,
+    pub task: TaskInQueue,
+    pub finalized_lock_attempts: Vec<LockAttempt>,
     pub is_reindexed: bool,
-    pub extra: T,
 }
 
-impl<T> ExecutionEnvironment<T> {
+impl ExecutionEnvironment {
     //fn new(cu: usize) -> Self {
     //    Self {
     //        cu,
@@ -113,8 +112,8 @@ impl<T> ExecutionEnvironment<T> {
 unsafe trait AtScheduleThread: Copy {}
 pub unsafe trait NotAtScheduleThread: Copy {}
 
-impl<T> PageRc<T> {
-    fn page_mut<AST: AtScheduleThread>(&self, _ast: AST) -> std::cell::RefMut<'_, Page<T>> {
+impl PageRc {
+    fn page_mut<AST: AtScheduleThread>(&self, _ast: AST) -> std::cell::RefMut<'_, Page> {
         self.0 .0.borrow_mut()
     }
 }
@@ -127,17 +126,17 @@ enum LockStatus {
 }
 
 #[derive(Debug)]
-pub struct LockAttempt<T> {
-    target: PageRc<T>,
+pub struct LockAttempt {
+    target: PageRc,
     status: LockStatus,
     requested_usage: RequestedUsage,
     //pub heaviest_uncontended: arc_swap::ArcSwapOption<Task>,
-    pub heaviest_uncontended: Option<TaskInQueue<T>>,
+    pub heaviest_uncontended: Option<TaskInQueue>,
     //remembered: bool,
 }
 
-impl<T> LockAttempt<T> {
-    pub fn new(target: PageRc<T>, requested_usage: RequestedUsage) -> Self {
+impl LockAttempt {
+    pub fn new(target: PageRc, requested_usage: RequestedUsage) -> Self {
         Self {
             target,
             status: LockStatus::Succeded,
@@ -157,7 +156,7 @@ impl<T> LockAttempt<T> {
         }
     }
 
-    pub fn contended_unique_weights<T>(&self) -> &TaskIds<T> {
+    pub fn contended_unique_weights(&self) -> &TaskIds {
         &self.target.0 .1
     }
 }
@@ -194,13 +193,13 @@ pub enum RequestedUsage {
 }
 
 #[derive(Debug, Default)]
-pub struct TaskIds<T> {
-    task_ids: crossbeam_skiplist::SkipMap<UniqueWeight, TaskInQueue<T>>,
+pub struct TaskIds {
+    task_ids: crossbeam_skiplist::SkipMap<UniqueWeight, TaskInQueue>,
 }
 
-impl<T> TaskIds<T> {
+impl TaskIds {
     #[inline(never)]
-    pub fn insert_task<T>(&self, u: TaskId, task: TaskInQueue<T>) {
+    pub fn insert_task(&self, u: TaskId, task: TaskInQueue) {
         let mut is_inserted = false;
         self.task_ids.get_or_insert_with(u, || {
             is_inserted = true;
@@ -216,25 +215,25 @@ impl<T> TaskIds<T> {
     }
 
     #[inline(never)]
-    pub fn heaviest_task_cursor<T>(
+    pub fn heaviest_task_cursor(
         &self,
-    ) -> Option<crossbeam_skiplist::map::Entry<'_, UniqueWeight, TaskInQueue<T>>> {
+    ) -> Option<crossbeam_skiplist::map::Entry<'_, UniqueWeight, TaskInQueue>> {
         self.task_ids.back()
     }
 }
 
 #[derive(Debug)]
-pub struct Page<T> {
+pub struct Page {
     current_usage: Usage,
     next_usage: Usage,
-    provisional_task_ids: Vec<triomphe::Arc<ProvisioningTracker<T>>>,
+    provisional_task_ids: Vec<triomphe::Arc<ProvisioningTracker>>,
     cu: CU,
     //loaded account from Accounts db
     //comulative_cu for qos; i.e. track serialized cumulative keyed by addresses and bail out block
     //producing as soon as any one of cu from the executing thread reaches to the limit
 }
 
-impl<T> Page<T> {
+impl Page {
     fn new(current_usage: Usage) -> Self {
         Self {
             current_usage,
@@ -250,32 +249,32 @@ impl<T> Page<T> {
     }
 }
 
-//type AddressMap = std::collections::HashMap<Pubkey, PageRc<T>>;
-type AddressMap<T> = std::sync::Arc<dashmap::DashMap<Pubkey, PageRc<T>>>;
+//type AddressMap = std::collections::HashMap<Pubkey, PageRc>;
+type AddressMap = std::sync::Arc<dashmap::DashMap<Pubkey, PageRc>>;
 type TaskId = UniqueWeight;
-type WeightedTaskIds<T> = std::collections::BTreeMap<TaskId, TaskInQueue<T>>;
+type WeightedTaskIds = std::collections::BTreeMap<TaskId, TaskInQueue>;
 //type AddressMapEntry<'a, K, V> = std::collections::hash_map::Entry<'a, K, V>;
-type AddressMapEntry<'a, T> = dashmap::mapref::entry::Entry<'a, Pubkey, PageRc<T>>;
+type AddressMapEntry<'a> = dashmap::mapref::entry::Entry<'a, Pubkey, PageRc>;
 
 type StuckTaskId = (CU, TaskId);
 
 // needs ttl mechanism and prune
 #[derive(Default)]
-pub struct AddressBook<T> {
-    book: AddressMap<T>,
-    uncontended_task_ids: WeightedTaskIds<T>,
-    fulfilled_provisional_task_ids: WeightedTaskIds<T>,
-    stuck_tasks: std::collections::BTreeMap<StuckTaskId, TaskInQueue<T>>,
+pub struct AddressBook {
+    book: AddressMap,
+    uncontended_task_ids: WeightedTaskIds,
+    fulfilled_provisional_task_ids: WeightedTaskIds,
+    stuck_tasks: std::collections::BTreeMap<StuckTaskId, TaskInQueue>,
 }
 
 #[derive(Debug)]
-struct ProvisioningTracker<T> {
+struct ProvisioningTracker {
     remaining_count: std::sync::atomic::AtomicUsize,
-    task: TaskInQueue<T>,
+    task: TaskInQueue,
 }
 
-impl<T> ProvisioningTracker<T> {
-    fn new(remaining_count: usize, task: TaskInQueue<T>) -> Self {
+impl ProvisioningTracker {
+    fn new(remaining_count: usize, task: TaskInQueue) -> Self {
         Self {
             remaining_count: std::sync::atomic::AtomicUsize::new(remaining_count),
             task,
@@ -301,14 +300,14 @@ impl<T> ProvisioningTracker<T> {
     }
 }
 
-impl<T> AddressBook<T> {
+impl AddressBook {
     #[inline(never)]
     fn attempt_lock_address<AST: AtScheduleThread>(
         ast: AST,
         from_runnable: bool,
         prefer_immediate: bool,
         unique_weight: &UniqueWeight,
-        attempt: &mut LockAttempt<T>,
+        attempt: &mut LockAttempt,
     ) -> CU {
         let LockAttempt {
             target,
@@ -379,7 +378,7 @@ impl<T> AddressBook<T> {
     fn reset_lock<AST: AtScheduleThread>(
         &mut self,
         ast: AST,
-        attempt: &mut LockAttempt<T>,
+        attempt: &mut LockAttempt,
         after_execution: bool,
     ) -> bool {
         match attempt.status {
@@ -399,7 +398,7 @@ impl<T> AddressBook<T> {
     }
 
     #[inline(never)]
-    fn unlock<AST: AtScheduleThread>(&mut self, ast: AST, attempt: &mut LockAttempt<T>) -> bool {
+    fn unlock<AST: AtScheduleThread>(&mut self, ast: AST, attempt: &mut LockAttempt) -> bool {
         //debug_assert!(attempt.is_success());
 
         let mut newly_uncontended = false;
@@ -434,7 +433,7 @@ impl<T> AddressBook<T> {
     }
 
     #[inline(never)]
-    fn cancel<AST: AtScheduleThread>(&mut self, ast: AST, attempt: &mut LockAttempt<T>) {
+    fn cancel<AST: AtScheduleThread>(&mut self, ast: AST, attempt: &mut LockAttempt) {
         let mut page = attempt.target.page_mut(ast);
 
         match page.next_usage {
@@ -448,7 +447,7 @@ impl<T> AddressBook<T> {
         }
     }
 
-    pub fn preloader(&self) -> Preloader<T> {
+    pub fn preloader(&self) -> Preloader {
         Preloader {
             book: std::sync::Arc::clone(&self.book),
         }
@@ -456,13 +455,13 @@ impl<T> AddressBook<T> {
 }
 
 #[derive(Debug)]
-pub struct Preloader<T> {
-    book: AddressMap<T>,
+pub struct Preloader {
+    book: AddressMap,
 }
 
-impl<T> Preloader<T> {
+impl Preloader {
     #[inline(never)]
-    pub fn load(&self, address: Pubkey) -> PageRc<T> {
+    pub fn load(&self, address: Pubkey) -> PageRc {
         PageRc::clone(&self.book.entry(address).or_insert_with(|| {
             PageRc(PageRcInner::new((
                 core::cell::RefCell::new(Page::new(Usage::unused())),
@@ -499,9 +498,9 @@ struct Bundle {
 }
 
 #[derive(Debug)]
-pub struct Task<T> {
+pub struct Task {
     unique_weight: UniqueWeight,
-    pub tx: (SanitizedTransaction, LockAttemptsInCell<T>), // actually should be Bundle
+    pub tx: (SanitizedTransaction, LockAttemptsInCell), // actually should be Bundle
     pub contention_count: std::sync::atomic::AtomicUsize,
     pub busiest_page_cu: std::sync::atomic::AtomicU64,
     pub uncontended: std::sync::atomic::AtomicUsize,
@@ -511,18 +510,17 @@ pub struct Task<T> {
     pub queue_end_time: std::sync::atomic::AtomicUsize,
     pub execute_time: std::sync::atomic::AtomicUsize,
     pub commit_time: std::sync::atomic::AtomicUsize,
-    pub for_indexer: LockAttemptsInCell<T>,
-    pub extra: T,
+    pub for_indexer: LockAttemptsInCell,
 }
 
 #[derive(Debug)]
-pub struct LockAttemptsInCell<T>(std::cell::RefCell<Vec<LockAttempt<T>>>);
+pub struct LockAttemptsInCell(std::cell::RefCell<Vec<LockAttempt>>);
 
-unsafe impl<T> Send for LockAttemptsInCell<T> {}
-unsafe impl<T> Sync for LockAttemptsInCell<T> {}
+unsafe impl Send for LockAttemptsInCell {}
+unsafe impl Sync for LockAttemptsInCell {}
 
-impl<T> LockAttemptsInCell<T> {
-    fn new(ll: std::cell::RefCell<Vec<LockAttempt<T>>>) -> Self {
+impl LockAttemptsInCell {
+    fn new(ll: std::cell::RefCell<Vec<LockAttempt>>) -> Self {
         Self(ll)
     }
 }
@@ -532,12 +530,12 @@ impl<T> LockAttemptsInCell<T> {
 // execute_time ---> exec clock
 // commit_time  -+
 
-impl<T> Task<T> {
+impl Task {
     pub fn new_for_queue<NAST: NotAtScheduleThread>(
         nast: NAST,
         unique_weight: UniqueWeight,
-        tx: (SanitizedTransaction, Vec<LockAttempt<T>>),
-    ) -> TaskInQueue<T> {
+        tx: (SanitizedTransaction, Vec<LockAttempt>),
+    ) -> TaskInQueue {
         TaskInQueue::new(Self {
             for_indexer: LockAttemptsInCell::new(std::cell::RefCell::new(
                 tx.1.iter().map(|a| a.clone_for_test()).collect(),
@@ -557,21 +555,21 @@ impl<T> Task<T> {
     }
 
     #[inline(never)]
-    pub fn clone_in_queue(this: &TaskInQueue<T>) -> TaskInQueue<T> {
-        TaskInQueue::<T>::clone(this)
+    pub fn clone_in_queue(this: &TaskInQueue) -> TaskInQueue {
+        TaskInQueue::clone(this)
     }
 
     fn lock_attempts_mut<AST: AtScheduleThread>(
         &self,
         _ast: AST,
-    ) -> std::cell::RefMut<'_, Vec<LockAttempt<T>>> {
+    ) -> std::cell::RefMut<'_, Vec<LockAttempt>> {
         self.tx.1 .0.borrow_mut()
     }
 
     fn lock_attempts_not_mut<NAST: NotAtScheduleThread>(
         &self,
         _nast: NAST,
-    ) -> std::cell::Ref<'_, Vec<LockAttempt<T>>> {
+    ) -> std::cell::Ref<'_, Vec<LockAttempt>> {
         self.tx.1 .0.borrow()
     }
 
@@ -701,8 +699,8 @@ impl<T> Task<T> {
 
     #[inline(never)]
     fn index_with_address_book(
-        this: &TaskInQueue<T>,
-        task_sender: &crossbeam_channel::Sender<(TaskInQueue<T>, Vec<LockAttempt<T>>)>,
+        this: &TaskInQueue,
+        task_sender: &crossbeam_channel::Sender<(TaskInQueue, Vec<LockAttempt>)>,
     ) {
         //for lock_attempt in self.lock_attempts_mut(ast).iter() {
         //    lock_attempt.contended_unique_weights().insert_task(unique_weight, Task::clone_in_queue(&self));
@@ -724,14 +722,14 @@ impl<T> Task<T> {
 
 // RunnableQueue, ContendedQueue?
 #[derive(Default, Debug, Clone)]
-pub struct TaskQueue<T> {
-    tasks: std::collections::BTreeMap<UniqueWeight, TaskInQueue<T>>,
+pub struct TaskQueue {
+    tasks: std::collections::BTreeMap<UniqueWeight, TaskInQueue>,
     //tasks: im::OrdMap<UniqueWeight, TaskInQueue>,
     //tasks: im::HashMap<UniqueWeight, TaskInQueue>,
     //tasks: std::sync::Arc<dashmap::DashMap<UniqueWeight, TaskInQueue>>,
 }
 
-pub type TaskInQueue<T> = triomphe::Arc<Task<T>>;
+pub type TaskInQueue = triomphe::Arc<Task>;
 
 //type TaskQueueEntry<'a> = im::ordmap::Entry<'a, UniqueWeight, TaskInQueue>;
 //type TaskQueueOccupiedEntry<'a> = im::ordmap::OccupiedEntry<'a, UniqueWeight, TaskInQueue>;
@@ -739,20 +737,20 @@ pub type TaskInQueue<T> = triomphe::Arc<Task<T>>;
 //type TaskQueueOccupiedEntry<'a> = im::hashmap::OccupiedEntry<'a, UniqueWeight, TaskInQueue, std::collections::hash_map::RandomState>;
 //type TaskQueueEntry<'a> = dashmap::mapref::entry::Entry<'a, UniqueWeight, TaskInQueue>;
 //type TaskQueueOccupiedEntry<'a> = dashmap::mapref::entry::OccupiedEntry<'a, UniqueWeight, TaskInQueue, std::collections::hash_map::RandomState>;
-type TaskQueueEntry<'a, T> = std::collections::btree_map::Entry<'a, UniqueWeight, TaskInQueue<T>>;
-type TaskQueueOccupiedEntry<'a, T> =
-    std::collections::btree_map::OccupiedEntry<'a, UniqueWeight, TaskInQueue<T>>;
+type TaskQueueEntry<'a> = std::collections::btree_map::Entry<'a, UniqueWeight, TaskInQueue>;
+type TaskQueueOccupiedEntry<'a> =
+    std::collections::btree_map::OccupiedEntry<'a, UniqueWeight, TaskInQueue>;
 
-impl<T> TaskQueue<T> {
+impl TaskQueue {
     #[inline(never)]
-    fn add_to_schedule(&mut self, unique_weight: UniqueWeight, task: TaskInQueue<T>) {
+    fn add_to_schedule(&mut self, unique_weight: UniqueWeight, task: TaskInQueue) {
         //trace!("TaskQueue::add(): {:?}", unique_weight);
         let pre_existed = self.tasks.insert(unique_weight, task);
         assert!(pre_existed.is_none()); //, "identical shouldn't exist: {:?}", unique_weight);
     }
 
     #[inline(never)]
-    fn heaviest_entry_to_execute(&mut self) -> Option<TaskQueueOccupiedEntry<'_, T>> {
+    fn heaviest_entry_to_execute(&mut self) -> Option<TaskQueueOccupiedEntry<'_>> {
         self.tasks.last_entry()
     }
 
@@ -762,14 +760,14 @@ impl<T> TaskQueue<T> {
 }
 
 #[inline(never)]
-fn attempt_lock_for_execution<'a, AST: AtScheduleThread, T>(
+fn attempt_lock_for_execution<'a, AST: AtScheduleThread>(
     ast: AST,
     from_runnable: bool,
     prefer_immediate: bool,
-    address_book: &mut AddressBook<T>,
+    address_book: &mut AddressBook,
     unique_weight: &UniqueWeight,
     message_hash: &'a Hash,
-    lock_attempts: &mut [LockAttempt<T>],
+    lock_attempts: &mut [LockAttempt],
 ) -> (usize, usize, CU) {
     // no short-cuircuit; we at least all need to add to the contended queue
     let mut unlockable_count = 0;
@@ -777,7 +775,7 @@ fn attempt_lock_for_execution<'a, AST: AtScheduleThread, T>(
     let mut busiest_page_cu = 1;
 
     for attempt in lock_attempts.iter_mut() {
-        let cu = AddressBook::<T>::attempt_lock_address(
+        let cu = AddressBook::attempt_lock_address(
             ast,
             from_runnable,
             prefer_immediate,
@@ -800,7 +798,7 @@ fn attempt_lock_for_execution<'a, AST: AtScheduleThread, T>(
     (unlockable_count, provisional_count, busiest_page_cu)
 }
 
-type PreprocessedTransaction<T> = (SanitizedTransaction, Vec<LockAttempt<T>>);
+type PreprocessedTransaction = (SanitizedTransaction, Vec<LockAttempt>);
 
 /*
 pub fn get_transaction_priority_details(tx: &SanitizedTransaction) -> u64 {
@@ -826,23 +824,23 @@ enum TaskSource {
 }
 
 impl ScheduleStage {
-    fn push_to_runnable_queue<T>(task: TaskInQueue<T>, runnable_queue: &mut TaskQueue<T>) {
+    fn push_to_runnable_queue(task: TaskInQueue, runnable_queue: &mut TaskQueue) {
         runnable_queue.add_to_schedule(task.unique_weight, task);
     }
 
     #[inline(never)]
-    fn get_heaviest_from_contended<'a, T>(
-        address_book: &'a mut AddressBook<T>,
-    ) -> Option<std::collections::btree_map::OccupiedEntry<'a, UniqueWeight, TaskInQueue<T>>> {
+    fn get_heaviest_from_contended<'a>(
+        address_book: &'a mut AddressBook,
+    ) -> Option<std::collections::btree_map::OccupiedEntry<'a, UniqueWeight, TaskInQueue>> {
         address_book.uncontended_task_ids.last_entry()
     }
 
     #[inline(never)]
-    fn select_next_task<'a, T>(
-        runnable_queue: &'a mut TaskQueue<T>,
-        address_book: &mut AddressBook<T>,
+    fn select_next_task<'a>(
+        runnable_queue: &'a mut TaskQueue,
+        address_book: &mut AddressBook,
         contended_count: &usize,
-    ) -> Option<(TaskSource, TaskInQueue<T>)> {
+    ) -> Option<(TaskSource, TaskInQueue)> {
         match (
             runnable_queue.heaviest_entry_to_execute(),
             Self::get_heaviest_from_contended(address_book),
@@ -899,17 +897,17 @@ impl ScheduleStage {
     }
 
     #[inline(never)]
-    fn pop_from_queue_then_lock<AST: AtScheduleThread, T>(
+    fn pop_from_queue_then_lock<AST: AtScheduleThread>(
         ast: AST,
-        task_sender: &crossbeam_channel::Sender<(TaskInQueue<T>, Vec<LockAttempt<T>>)>,
-        runnable_queue: &mut TaskQueue<T>,
-        address_book: &mut AddressBook<T>,
+        task_sender: &crossbeam_channel::Sender<(TaskInQueue, Vec<LockAttempt>)>,
+        runnable_queue: &mut TaskQueue,
+        address_book: &mut AddressBook,
         contended_count: &mut usize,
         prefer_immediate: bool,
         sequence_clock: &usize,
         queue_clock: &mut usize,
         provisioning_tracker_count: &mut usize,
-    ) -> Option<(UniqueWeight, TaskInQueue<T>, Vec<LockAttempt<T>>)> {
+    ) -> Option<(UniqueWeight, TaskInQueue, Vec<LockAttempt>)> {
         if let Some(mut a) = address_book.fulfilled_provisional_task_ids.pop_last() {
             trace!(
                 "expediate pop from provisional queue [rest: {}]",
@@ -1077,11 +1075,11 @@ impl ScheduleStage {
     }
 
     #[inline(never)]
-    fn finalize_lock_for_provisional_execution<AST: AtScheduleThread, T>(
+    fn finalize_lock_for_provisional_execution<AST: AtScheduleThread>(
         ast: AST,
-        address_book: &mut AddressBook<T>,
-        next_task: &Task<T>,
-        tracker: triomphe::Arc<ProvisioningTracker<T>>,
+        address_book: &mut AddressBook,
+        next_task: &Task,
+        tracker: triomphe::Arc<ProvisioningTracker>,
     ) {
         for l in next_task.lock_attempts_mut(ast).iter_mut() {
             match l.status {
@@ -1103,11 +1101,11 @@ impl ScheduleStage {
     }
 
     #[inline(never)]
-    fn reset_lock_for_failed_execution<AST: AtScheduleThread, T>(
+    fn reset_lock_for_failed_execution<AST: AtScheduleThread>(
         ast: AST,
-        address_book: &mut AddressBook<T>,
+        address_book: &mut AddressBook,
         unique_weight: &UniqueWeight,
-        lock_attempts: &mut [LockAttempt<T>],
+        lock_attempts: &mut [LockAttempt],
     ) {
         for l in lock_attempts {
             address_book.reset_lock(ast, l, false);
@@ -1115,10 +1113,10 @@ impl ScheduleStage {
     }
 
     #[inline(never)]
-    fn unlock_after_execution<AST: AtScheduleThread, T>(
+    fn unlock_after_execution<AST: AtScheduleThread>(
         ast: AST,
-        address_book: &mut AddressBook<T>,
-        lock_attempts: &mut [LockAttempt<T>],
+        address_book: &mut AddressBook,
+        lock_attempts: &mut [LockAttempt],
         provisioning_tracker_count: &mut usize,
         cu: CU,
     ) {
@@ -1191,19 +1189,19 @@ impl ScheduleStage {
                 }
             }
 
-            // todo: mem::forget and panic in LockAttempt<T>::drop()
+            // todo: mem::forget and panic in LockAttempt::drop()
         }
     }
 
     #[inline(never)]
-    fn prepare_scheduled_execution<T>(
-        address_book: &mut AddressBook<T>,
+    fn prepare_scheduled_execution(
+        address_book: &mut AddressBook,
         unique_weight: UniqueWeight,
-        task: TaskInQueue<T>,
-        finalized_lock_attempts: Vec<LockAttempt<T>>,
+        task: TaskInQueue,
+        finalized_lock_attempts: Vec<LockAttempt>,
         queue_clock: &usize,
         execute_clock: &mut usize,
-    ) -> Box<ExecutionEnvironment<T>> {
+    ) -> Box<ExecutionEnvironment> {
         let mut rng = rand::thread_rng();
         // load account now from AccountsDb
         task.record_execute_time(*queue_clock, *execute_clock);
@@ -1219,10 +1217,10 @@ impl ScheduleStage {
     }
 
     #[inline(never)]
-    fn commit_completed_execution<AST: AtScheduleThread, T>(
+    fn commit_completed_execution<AST: AtScheduleThread>(
         ast: AST,
-        ee: &mut ExecutionEnvironment<T>,
-        address_book: &mut AddressBook<T>,
+        ee: &mut ExecutionEnvironment,
+        address_book: &mut AddressBook,
         commit_time: &mut usize,
         provisioning_tracker_count: &mut usize,
     ) {
@@ -1257,18 +1255,18 @@ impl ScheduleStage {
     }
 
     #[inline(never)]
-    fn schedule_next_execution<AST: AtScheduleThread, T>(
+    fn schedule_next_execution<AST: AtScheduleThread>(
         ast: AST,
-        task_sender: &crossbeam_channel::Sender<(TaskInQueue<T>, Vec<LockAttempt<T>>)>,
-        runnable_queue: &mut TaskQueue<T>,
-        address_book: &mut AddressBook<T>,
+        task_sender: &crossbeam_channel::Sender<(TaskInQueue, Vec<LockAttempt>)>,
+        runnable_queue: &mut TaskQueue,
+        address_book: &mut AddressBook,
         contended_count: &mut usize,
         prefer_immediate: bool,
         sequence_time: &usize,
         queue_clock: &mut usize,
         execute_clock: &mut usize,
         provisioning_tracker_count: &mut usize,
-    ) -> Option<Box<ExecutionEnvironment<T>>> {
+    ) -> Option<Box<ExecutionEnvironment>> {
         let maybe_ee = Self::pop_from_queue_then_lock(
             ast,
             task_sender,
@@ -1287,9 +1285,9 @@ impl ScheduleStage {
     }
 
     #[inline(never)]
-    fn register_runnable_task<T>(
-        weighted_tx: TaskInQueue<T>,
-        runnable_queue: &mut TaskQueue<T>,
+    fn register_runnable_task(
+        weighted_tx: TaskInQueue,
+        runnable_queue: &mut TaskQueue,
         sequence_time: &mut usize,
     ) {
         weighted_tx.record_sequence_time(*sequence_time);
@@ -1297,15 +1295,15 @@ impl ScheduleStage {
         Self::push_to_runnable_queue(weighted_tx, runnable_queue)
     }
 
-    fn _run<AST: AtScheduleThread, T>(
+    fn _run<AST: AtScheduleThread>(
         ast: AST,
         max_executing_queue_count: usize,
-        runnable_queue: &mut TaskQueue<T>,
-        address_book: &mut AddressBook<T>,
-        from_prev: &crossbeam_channel::Receiver<SchedulablePayload<T>>,
-        to_execute_substage: &crossbeam_channel::Sender<ExecutablePayload<T>>,
-        from_exec: &crossbeam_channel::Receiver<UnlockablePayload<T>>,
-        maybe_to_next_stage: Option<&crossbeam_channel::Sender<DroppablePayload<T>>>, // assume nonblocking
+        runnable_queue: &mut TaskQueue,
+        address_book: &mut AddressBook,
+        from_prev: &crossbeam_channel::Receiver<SchedulablePayload>,
+        to_execute_substage: &crossbeam_channel::Sender<ExecutablePayload>,
+        from_exec: &crossbeam_channel::Receiver<UnlockablePayload>,
+        maybe_to_next_stage: Option<&crossbeam_channel::Sender<DroppablePayload>>, // assume nonblocking
     ) {
         let random_id = rand::thread_rng().gen::<u64>();
         info!("schedule_once:initial id_{:016x}", random_id);
@@ -1347,7 +1345,7 @@ impl ScheduleStage {
             (&ee_sender, Some(h))
         };
         let (task_sender, task_receiver) =
-            crossbeam_channel::unbounded::<(TaskInQueue, Vec<LockAttempt<T>>)>();
+            crossbeam_channel::unbounded::<(TaskInQueue, Vec<LockAttempt>)>();
         let indexer_count = std::env::var("INDEXER_COUNT")
             .unwrap_or(format!("{}", 4))
             .parse::<usize>()
@@ -1497,14 +1495,14 @@ impl ScheduleStage {
         info!("schedule_once:final id_{:016x} (from_disconnected: {}, from_exec_disconnected: {}, no_more_work: {}) (from: {}, to: {}, runnnable: {}, contended: {}, (immediate+provisional)/max: ({}+{})/{}) active from contended: {} stuck: {} completed: {}!", random_id, from_disconnected, from_exec_disconnected, no_more_work, from_prev.len(), to_execute_substage.len(), runnable_queue.task_count(), contended_count, executing_queue_count, provisioning_tracker_count, max_executing_queue_count, address_book.uncontended_task_ids.len(), address_book.stuck_tasks.len(), completed_count);
     }
 
-    pub fn run<T>(
+    pub fn run(
         max_executing_queue_count: usize,
-        runnable_queue: &mut TaskQueue<T>,
-        address_book: &mut AddressBook<T>,
-        from: &crossbeam_channel::Receiver<SchedulablePayload<T>>,
-        to_execute_substage: &crossbeam_channel::Sender<ExecutablePayload<T>>,
-        from_execute_substage: &crossbeam_channel::Receiver<UnlockablePayload<T>>,
-        maybe_to_next_stage: Option<&crossbeam_channel::Sender<DroppablePayload<T>>>, // assume nonblocking
+        runnable_queue: &mut TaskQueue,
+        address_book: &mut AddressBook,
+        from: &crossbeam_channel::Receiver<SchedulablePayload>,
+        to_execute_substage: &crossbeam_channel::Sender<ExecutablePayload>,
+        from_execute_substage: &crossbeam_channel::Receiver<UnlockablePayload>,
+        maybe_to_next_stage: Option<&crossbeam_channel::Sender<DroppablePayload>>, // assume nonblocking
     ) {
         #[derive(Clone, Copy, Debug)]
         struct AtTopOfScheduleThread;
@@ -1523,10 +1521,10 @@ impl ScheduleStage {
     }
 }
 
-pub struct SchedulablePayload<T>(pub TaskInQueue<T>);
-pub struct ExecutablePayload<T>(pub Box<ExecutionEnvironment<T>>);
-pub struct UnlockablePayload<T>(pub Box<ExecutionEnvironment<T>>);
-pub struct DroppablePayload<T>(pub Box<ExecutionEnvironment<T>>);
+pub struct SchedulablePayload(pub TaskInQueue);
+pub struct ExecutablePayload(pub Box<ExecutionEnvironment>);
+pub struct UnlockablePayload(pub Box<ExecutionEnvironment>);
+pub struct DroppablePayload(pub Box<ExecutionEnvironment>);
 
 struct ExecuteStage {
     //bank: Bank,

From 2513073b75de31221c8bd7a0357d2146575d4bd9 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 5 Sep 2022 21:31:24 +0900
Subject: [PATCH 2096/3199] save

---
 runtime/src/bank.rs | 28 ++++++++++++++++++++++++++++
 1 file changed, 28 insertions(+)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index acaea8ec00acb6..a82985f01cc5ef 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1240,11 +1240,39 @@ impl Default for Scheduler {
         let (completed_ee_sender, completed_ee_receiver) = crossbeam_channel::unbounded();
         let (droppable_ee_sender, droppable_ee_receiver) = crossbeam_channel::unbounded();
 
+        let bank = Arc::new(Mutex::new(None::<Option<Arc<Bank>>>));
+
         let executing_thread_handles = (0..8).map(|thx| {
             let (scheduled_ee_receiver, completed_ee_sender) = (scheduled_ee_receiver.clone(), completed_ee_sender.clone());
 
             std::thread::Builder::new().name(format!("solExec{:02}", thx)).spawn(move || {
             while let Ok(solana_scheduler::ExecutablePayload(mut ee)) = scheduled_ee_receiver.recv() {
+                let tx_account_lock_limit = bank.get_transaction_account_lock_limit();
+                let lock_result = ee.transaction
+                    .get_account_locks(tx_account_lock_limit)
+                    .map(|_| ());
+                let mut batch =
+                    TransactionBatch::new(vec![lock_result], ee.bank(), Cow::Owned(vec![ee.transaction]));
+                batch.set_needs_unlock(false);
+
+
+                let (tx_results, _balances) = bank.load_execute_and_commit_transactions(
+                    batch,
+                    MAX_PROCESSING_AGE,
+                    false,
+                    false,
+                    false,
+                    false,
+                    Default::default(),
+                    None
+                );
+                let TransactionResults {
+                    fee_collection_results,
+                    ..
+                } = tx_results;
+                ee.execution_result = fee_collection_results.iter().collect::<Result<_>>();
+
+
                 ee.reindex_with_address_book();
                 completed_ee_sender.send(solana_scheduler::UnlockablePayload(ee)).unwrap();
             }

From b62d4327740862ad23e14f236df1ffff87cbb3df Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 5 Sep 2022 21:31:44 +0900
Subject: [PATCH 2097/3199] save

---
 runtime/src/bank.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index a82985f01cc5ef..4fd2e292b2ea3e 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1240,7 +1240,7 @@ impl Default for Scheduler {
         let (completed_ee_sender, completed_ee_receiver) = crossbeam_channel::unbounded();
         let (droppable_ee_sender, droppable_ee_receiver) = crossbeam_channel::unbounded();
 
-        let bank = Arc::new(Mutex::new(None::<Option<Arc<Bank>>>));
+        let bank = Arc::new(std::sync::Mutex::new(None::<Option<Arc<Bank>>>));
 
         let executing_thread_handles = (0..8).map(|thx| {
             let (scheduled_ee_receiver, completed_ee_sender) = (scheduled_ee_receiver.clone(), completed_ee_sender.clone());

From 23724cdd70cb6d18c7cc11c5f65c4bd0e2568335 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 5 Sep 2022 21:32:10 +0900
Subject: [PATCH 2098/3199] save

---
 runtime/src/bank.rs | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 4fd2e292b2ea3e..424b3c97c1a705 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1247,6 +1247,8 @@ impl Default for Scheduler {
 
             std::thread::Builder::new().name(format!("solExec{:02}", thx)).spawn(move || {
             while let Ok(solana_scheduler::ExecutablePayload(mut ee)) = scheduled_ee_receiver.recv() {
+                let bank = bank.read().unwrap().unwrap();
+
                 let tx_account_lock_limit = bank.get_transaction_account_lock_limit();
                 let lock_result = ee.transaction
                     .get_account_locks(tx_account_lock_limit)

From 83e0278c32a35bb9c48a139d910c9f64e6a4e1e8 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 5 Sep 2022 21:32:27 +0900
Subject: [PATCH 2099/3199] save

---
 runtime/src/bank.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 424b3c97c1a705..aed43130fd13da 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1254,7 +1254,7 @@ impl Default for Scheduler {
                     .get_account_locks(tx_account_lock_limit)
                     .map(|_| ());
                 let mut batch =
-                    TransactionBatch::new(vec![lock_result], ee.bank(), Cow::Owned(vec![ee.transaction]));
+                    TransactionBatch::new(vec![lock_result], bank, Cow::Owned(vec![ee.transaction]));
                 batch.set_needs_unlock(false);
 
 

From ec4ba8c596547d5df9ab7759a26698590f268a53 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 5 Sep 2022 21:32:52 +0900
Subject: [PATCH 2100/3199] save

---
 runtime/src/bank.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index aed43130fd13da..156335910ec097 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1254,7 +1254,7 @@ impl Default for Scheduler {
                     .get_account_locks(tx_account_lock_limit)
                     .map(|_| ());
                 let mut batch =
-                    TransactionBatch::new(vec![lock_result], bank, Cow::Owned(vec![ee.transaction]));
+                    TransactionBatch::new(vec![lock_result], bank, Cow::Owned(vec![ee.task.tx.0]));
                 batch.set_needs_unlock(false);
 
 

From 07f3c8894813eb5aceea696a625728a8db2fa5ce Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 5 Sep 2022 21:33:20 +0900
Subject: [PATCH 2101/3199] save

---
 runtime/src/bank.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 156335910ec097..ffbe46d0966e21 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1240,7 +1240,7 @@ impl Default for Scheduler {
         let (completed_ee_sender, completed_ee_receiver) = crossbeam_channel::unbounded();
         let (droppable_ee_sender, droppable_ee_receiver) = crossbeam_channel::unbounded();
 
-        let bank = Arc::new(std::sync::Mutex::new(None::<Option<Arc<Bank>>>));
+        let bank = Arc::new(std::sync::RwLock::new(None::<Option<Arc<Bank>>>));
 
         let executing_thread_handles = (0..8).map(|thx| {
             let (scheduled_ee_receiver, completed_ee_sender) = (scheduled_ee_receiver.clone(), completed_ee_sender.clone());

From 000ae7e82a18f07e17c522ff6265f49368484552 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 5 Sep 2022 21:33:59 +0900
Subject: [PATCH 2102/3199] save

---
 runtime/src/bank.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index ffbe46d0966e21..491aac94cc4c7d 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1247,10 +1247,10 @@ impl Default for Scheduler {
 
             std::thread::Builder::new().name(format!("solExec{:02}", thx)).spawn(move || {
             while let Ok(solana_scheduler::ExecutablePayload(mut ee)) = scheduled_ee_receiver.recv() {
-                let bank = bank.read().unwrap().unwrap();
+                let bank = bank.read().unwrap().unwrap().unwrap();
 
                 let tx_account_lock_limit = bank.get_transaction_account_lock_limit();
-                let lock_result = ee.transaction
+                let lock_result = ee.task.tx.0
                     .get_account_locks(tx_account_lock_limit)
                     .map(|_| ());
                 let mut batch =

From 25a58c4f89cf2737441fd6433ac3bcb91d471052 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 5 Sep 2022 21:34:54 +0900
Subject: [PATCH 2103/3199] save

---
 runtime/src/bank.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 491aac94cc4c7d..f44aa70c70b768 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1272,7 +1272,7 @@ impl Default for Scheduler {
                     fee_collection_results,
                     ..
                 } = tx_results;
-                ee.execution_result = fee_collection_results.iter().collect::<Result<_>>();
+                ee.execution_result = fee_collection_results.into_iter().copied().collect::<Result<_>>();
 
 
                 ee.reindex_with_address_book();

From 86b0066261c30de696d4b0dde92731001f7c21b4 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 5 Sep 2022 21:35:12 +0900
Subject: [PATCH 2104/3199] save

---
 runtime/src/bank.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index f44aa70c70b768..724419969ca455 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1272,7 +1272,7 @@ impl Default for Scheduler {
                     fee_collection_results,
                     ..
                 } = tx_results;
-                ee.execution_result = fee_collection_results.into_iter().copied().collect::<Result<_>>();
+                ee.execution_result = fee_collection_results.into_iter().collect::<Result<_>>();
 
 
                 ee.reindex_with_address_book();

From 3d8818da451b8d1ea0e73a52a62b4845080f4c36 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 5 Sep 2022 21:35:33 +0900
Subject: [PATCH 2105/3199] save

---
 runtime/src/bank.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 724419969ca455..4d3931d40fb7ef 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1259,7 +1259,7 @@ impl Default for Scheduler {
 
 
                 let (tx_results, _balances) = bank.load_execute_and_commit_transactions(
-                    batch,
+                    &batch,
                     MAX_PROCESSING_AGE,
                     false,
                     false,

From a384d25cc2deab35095c0fff1830a57b36341c68 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 5 Sep 2022 21:36:08 +0900
Subject: [PATCH 2106/3199] save

---
 runtime/src/bank.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 4d3931d40fb7ef..ecf23ec8240bfa 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1254,7 +1254,7 @@ impl Default for Scheduler {
                     .get_account_locks(tx_account_lock_limit)
                     .map(|_| ());
                 let mut batch =
-                    TransactionBatch::new(vec![lock_result], bank, Cow::Owned(vec![ee.task.tx.0]));
+                    TransactionBatch::new(vec![lock_result], &bank, Cow::Owned(vec![ee.task.tx.0]));
                 batch.set_needs_unlock(false);
 
 
@@ -1265,7 +1265,7 @@ impl Default for Scheduler {
                     false,
                     false,
                     false,
-                    Default::default(),
+                    &mut Default::default(),
                     None
                 );
                 let TransactionResults {

From 0fb57b9c9b5322d80a338e03e5e153dfb9dd5871 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 5 Sep 2022 21:36:32 +0900
Subject: [PATCH 2107/3199] save

---
 scheduler/src/lib.rs | 1 +
 1 file changed, 1 insertion(+)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index b5a1d91d31a84c..a4dd959e1164b4 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -36,6 +36,7 @@ pub struct ExecutionEnvironment {
     pub task: TaskInQueue,
     pub finalized_lock_attempts: Vec<LockAttempt>,
     pub is_reindexed: bool,
+    pub execution_result: usize,
 }
 
 impl ExecutionEnvironment {

From 12f038c49f9b792123365797a454b1cde48fe7bd Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 5 Sep 2022 21:36:57 +0900
Subject: [PATCH 2108/3199] save

---
 scheduler/src/lib.rs | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index a4dd959e1164b4..da53ea80f64108 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -36,7 +36,7 @@ pub struct ExecutionEnvironment {
     pub task: TaskInQueue,
     pub finalized_lock_attempts: Vec<LockAttempt>,
     pub is_reindexed: bool,
-    pub execution_result: usize,
+    pub execution_result: Option<usize>,
 }
 
 impl ExecutionEnvironment {
@@ -1214,6 +1214,7 @@ impl ScheduleStage {
             cu: rng.gen_range(3, 1000),
             finalized_lock_attempts,
             is_reindexed: Default::default(),
+            execution_result: Default::default(),
         })
     }
 

From c3da6d56f312ddf0f968b4a5926ca302de27ed80 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 5 Sep 2022 21:37:22 +0900
Subject: [PATCH 2109/3199] save

---
 runtime/src/bank.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index ecf23ec8240bfa..09ca78e9bbb0a4 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1272,7 +1272,7 @@ impl Default for Scheduler {
                     fee_collection_results,
                     ..
                 } = tx_results;
-                ee.execution_result = fee_collection_results.into_iter().collect::<Result<_>>();
+                ee.execution_result = Some(fee_collection_results.into_iter().collect::<Result<_>>());
 
 
                 ee.reindex_with_address_book();

From 2234ec0a523a378302e7840e70355da26eb2684c Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 5 Sep 2022 21:38:03 +0900
Subject: [PATCH 2110/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index da53ea80f64108..694b1cfd7373a2 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -36,7 +36,7 @@ pub struct ExecutionEnvironment {
     pub task: TaskInQueue,
     pub finalized_lock_attempts: Vec<LockAttempt>,
     pub is_reindexed: bool,
-    pub execution_result: Option<usize>,
+    pub execution_result: Option<Result<(), TransactionError>>,
 }
 
 impl ExecutionEnvironment {

From 2f2e3bea0ed021efc5b7dd2241f3e7f262342ea0 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 5 Sep 2022 21:38:53 +0900
Subject: [PATCH 2111/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 694b1cfd7373a2..180cb1ab02d5dc 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -36,7 +36,7 @@ pub struct ExecutionEnvironment {
     pub task: TaskInQueue,
     pub finalized_lock_attempts: Vec<LockAttempt>,
     pub is_reindexed: bool,
-    pub execution_result: Option<Result<(), TransactionError>>,
+    pub execution_result: Option<Result<(), solana_sdk::transaction::TransactionError>>,
 }
 
 impl ExecutionEnvironment {

From fafa1c148a14ef4de3e1611d68aa6fa0b922fd70 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 5 Sep 2022 21:40:05 +0900
Subject: [PATCH 2112/3199] save

---
 runtime/src/bank.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 09ca78e9bbb0a4..6bdb01e8468e45 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1247,7 +1247,7 @@ impl Default for Scheduler {
 
             std::thread::Builder::new().name(format!("solExec{:02}", thx)).spawn(move || {
             while let Ok(solana_scheduler::ExecutablePayload(mut ee)) = scheduled_ee_receiver.recv() {
-                let bank = bank.read().unwrap().unwrap().unwrap();
+                let bank = bank.read().unwrap().unwrap();
 
                 let tx_account_lock_limit = bank.get_transaction_account_lock_limit();
                 let lock_result = ee.task.tx.0

From 418c1dbc634af6912e7c74558ec9ce7409244ced Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 5 Sep 2022 21:40:27 +0900
Subject: [PATCH 2113/3199] save

---
 runtime/src/bank.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 6bdb01e8468e45..05050c705d3496 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1247,7 +1247,7 @@ impl Default for Scheduler {
 
             std::thread::Builder::new().name(format!("solExec{:02}", thx)).spawn(move || {
             while let Ok(solana_scheduler::ExecutablePayload(mut ee)) = scheduled_ee_receiver.recv() {
-                let bank = bank.read().unwrap().unwrap();
+                let bank = bank.read().unwrap();
 
                 let tx_account_lock_limit = bank.get_transaction_account_lock_limit();
                 let lock_result = ee.task.tx.0

From 783f825ae5c85819056567402f04bf40cb704311 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 5 Sep 2022 21:40:48 +0900
Subject: [PATCH 2114/3199] save

---
 runtime/src/bank.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 05050c705d3496..349b79a39e2dcf 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1247,7 +1247,7 @@ impl Default for Scheduler {
 
             std::thread::Builder::new().name(format!("solExec{:02}", thx)).spawn(move || {
             while let Ok(solana_scheduler::ExecutablePayload(mut ee)) = scheduled_ee_receiver.recv() {
-                let bank = bank.read().unwrap();
+                let bank = bank.read().unwrap().as_ref().unwrap();
 
                 let tx_account_lock_limit = bank.get_transaction_account_lock_limit();
                 let lock_result = ee.task.tx.0

From 3c18df51167bc986a4a0faba56d7afa3a287c5b8 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 5 Sep 2022 21:41:08 +0900
Subject: [PATCH 2115/3199] save

---
 runtime/src/bank.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 349b79a39e2dcf..90a2148be1a9a9 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1247,7 +1247,7 @@ impl Default for Scheduler {
 
             std::thread::Builder::new().name(format!("solExec{:02}", thx)).spawn(move || {
             while let Ok(solana_scheduler::ExecutablePayload(mut ee)) = scheduled_ee_receiver.recv() {
-                let bank = bank.read().unwrap().as_ref().unwrap();
+                let bank = bank.read().unwrap().unwrap().as_ref();
 
                 let tx_account_lock_limit = bank.get_transaction_account_lock_limit();
                 let lock_result = ee.task.tx.0

From 9b7baaad5d1af27b25b77aa4047d7db58c1866c6 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 5 Sep 2022 21:41:24 +0900
Subject: [PATCH 2116/3199] save

---
 runtime/src/bank.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 90a2148be1a9a9..a894b75210da8c 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1247,7 +1247,7 @@ impl Default for Scheduler {
 
             std::thread::Builder::new().name(format!("solExec{:02}", thx)).spawn(move || {
             while let Ok(solana_scheduler::ExecutablePayload(mut ee)) = scheduled_ee_receiver.recv() {
-                let bank = bank.read().unwrap().unwrap().as_ref();
+                let bank = bank.read().unwrap().unwrap().as_ref().unwrap();
 
                 let tx_account_lock_limit = bank.get_transaction_account_lock_limit();
                 let lock_result = ee.task.tx.0

From c964a73b43c6e4662136c9f23e7edf2a4a66a885 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 5 Sep 2022 21:42:03 +0900
Subject: [PATCH 2117/3199] save

---
 runtime/src/bank.rs | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index a894b75210da8c..4f3e8eca26881b 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1247,7 +1247,8 @@ impl Default for Scheduler {
 
             std::thread::Builder::new().name(format!("solExec{:02}", thx)).spawn(move || {
             while let Ok(solana_scheduler::ExecutablePayload(mut ee)) = scheduled_ee_receiver.recv() {
-                let bank = bank.read().unwrap().unwrap().as_ref().unwrap();
+                let bank_r = bank.read().unwrap().unwrap();
+                let bank = bank_r.as_ref().unwrap();
 
                 let tx_account_lock_limit = bank.get_transaction_account_lock_limit();
                 let lock_result = ee.task.tx.0

From 524c304efb897981b1312642addda0b15e4344fe Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 5 Sep 2022 21:42:52 +0900
Subject: [PATCH 2118/3199] save

---
 runtime/src/bank.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 4f3e8eca26881b..154be5bc64f6e7 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1247,8 +1247,8 @@ impl Default for Scheduler {
 
             std::thread::Builder::new().name(format!("solExec{:02}", thx)).spawn(move || {
             while let Ok(solana_scheduler::ExecutablePayload(mut ee)) = scheduled_ee_receiver.recv() {
-                let bank_r = bank.read().unwrap().unwrap();
-                let bank = bank_r.as_ref().unwrap();
+                let bank_r = bank.read().unwrap();
+                let bank = (&bank_r).as_ref().unwrap();
 
                 let tx_account_lock_limit = bank.get_transaction_account_lock_limit();
                 let lock_result = ee.task.tx.0

From a81e53759448120d7113cee5377ff7b277c18acc Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 5 Sep 2022 21:43:25 +0900
Subject: [PATCH 2119/3199] save

---
 runtime/src/bank.rs | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 154be5bc64f6e7..180b273bd01c7f 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1248,7 +1248,8 @@ impl Default for Scheduler {
             std::thread::Builder::new().name(format!("solExec{:02}", thx)).spawn(move || {
             while let Ok(solana_scheduler::ExecutablePayload(mut ee)) = scheduled_ee_receiver.recv() {
                 let bank_r = bank.read().unwrap();
-                let bank = (&bank_r).as_ref().unwrap();
+                let bank_o = (&bank_r).as_ref().unwrap();
+                let bank = bank_o.unwrap();
 
                 let tx_account_lock_limit = bank.get_transaction_account_lock_limit();
                 let lock_result = ee.task.tx.0

From 58c0430f83752479edf71b18fb0453b3c6d74c85 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 5 Sep 2022 21:43:55 +0900
Subject: [PATCH 2120/3199] save

---
 runtime/src/bank.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 180b273bd01c7f..6502bf561b3679 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1249,7 +1249,7 @@ impl Default for Scheduler {
             while let Ok(solana_scheduler::ExecutablePayload(mut ee)) = scheduled_ee_receiver.recv() {
                 let bank_r = bank.read().unwrap();
                 let bank_o = (&bank_r).as_ref().unwrap();
-                let bank = bank_o.unwrap();
+                let bank = bank_o.as_ref().unwrap();
 
                 let tx_account_lock_limit = bank.get_transaction_account_lock_limit();
                 let lock_result = ee.task.tx.0

From 4eb416f70ed9aec6d074ed15e2e20b694040ee08 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 5 Sep 2022 21:44:19 +0900
Subject: [PATCH 2121/3199] save

---
 runtime/src/bank.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 6502bf561b3679..e8ea3b46e64277 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1256,7 +1256,7 @@ impl Default for Scheduler {
                     .get_account_locks(tx_account_lock_limit)
                     .map(|_| ());
                 let mut batch =
-                    TransactionBatch::new(vec![lock_result], &bank, Cow::Owned(vec![ee.task.tx.0]));
+                    TransactionBatch::new(vec![lock_result], &bank, Cow::Owned(vec![ee.task.tx.0.clone()]));
                 batch.set_needs_unlock(false);
 
 

From 6341c9b6e855291cb2e13fa55d0fe529012ebe86 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 5 Sep 2022 21:45:20 +0900
Subject: [PATCH 2122/3199] save

---
 runtime/src/bank.rs | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index e8ea3b46e64277..471f2d220ea1d8 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1205,6 +1205,7 @@ struct Scheduler {
     preloader: Arc<solana_scheduler::Preloader>,
     graceful_stop_initiated: bool,
     errors: Arc<std::sync::Mutex<Vec<Result<()>>>>,
+    bank: usize,
 }
 
 impl Scheduler {
@@ -1322,6 +1323,7 @@ impl Default for Scheduler {
             preloader,
             graceful_stop_initiated: Default::default(),
             errors,
+            bank,
         }
     }
 }

From d50bc3670bd7b4162167ef78d96efc77d9cb76a0 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 5 Sep 2022 21:45:39 +0900
Subject: [PATCH 2123/3199] save

---
 runtime/src/bank.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 471f2d220ea1d8..cefbb1fda669b8 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1205,7 +1205,7 @@ struct Scheduler {
     preloader: Arc<solana_scheduler::Preloader>,
     graceful_stop_initiated: bool,
     errors: Arc<std::sync::Mutex<Vec<Result<()>>>>,
-    bank: usize,
+    bank: std::sync::Arc<std::sync::RwLock<std::option::Option<std::option::Option<std::sync::Arc<Bank>>>>>,
 }
 
 impl Scheduler {

From b66c274179da6de84c08cca48173a5a20416cacd Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 5 Sep 2022 21:46:16 +0900
Subject: [PATCH 2124/3199] save

---
 runtime/src/bank.rs | 1 +
 1 file changed, 1 insertion(+)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index cefbb1fda669b8..c187782c3edfdb 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1245,6 +1245,7 @@ impl Default for Scheduler {
 
         let executing_thread_handles = (0..8).map(|thx| {
             let (scheduled_ee_receiver, completed_ee_sender) = (scheduled_ee_receiver.clone(), completed_ee_sender.clone());
+            let bank = bank.clone();
 
             std::thread::Builder::new().name(format!("solExec{:02}", thx)).spawn(move || {
             while let Ok(solana_scheduler::ExecutablePayload(mut ee)) = scheduled_ee_receiver.recv() {

From 1be9aae65e4c28ce4611aac0bba3711b54cf9c7f Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 5 Sep 2022 21:47:28 +0900
Subject: [PATCH 2125/3199] save

---
 runtime/src/bank.rs | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index c187782c3edfdb..ec9532a271d0c4 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -6380,6 +6380,8 @@ impl Bank {
         transaction_indexes: &Vec<usize>,
     ) {
         let scheduler = self.scheduler.read().unwrap();
+        if scheduler.bank.read().unwrap().is_none() {
+        }
         for (st, &i) in batch.sanitized_transactions().iter().zip(transaction_indexes.iter()) {
             scheduler.schedule(st, i);
         }

From f4768135c61afde68b697dbb59036b06f7f1941e Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 5 Sep 2022 21:48:22 +0900
Subject: [PATCH 2126/3199] save

---
 runtime/src/bank.rs | 11 +++++++++--
 1 file changed, 9 insertions(+), 2 deletions(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index ec9532a271d0c4..107da41ff03eff 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -6379,8 +6379,15 @@ impl Bank {
         batch: &TransactionBatch,
         transaction_indexes: &Vec<usize>,
     ) {
-        let scheduler = self.scheduler.read().unwrap();
-        if scheduler.bank.read().unwrap().is_none() {
+        let scheduler = {
+            let r = self.scheduler.read().unwrap();
+
+            if scheduler.bank.read().unwrap().is_none() {
+                drop(r);
+                panic!();
+            } else {
+                r
+            }
         }
         for (st, &i) in batch.sanitized_transactions().iter().zip(transaction_indexes.iter()) {
             scheduler.schedule(st, i);

From 898767ccd9602a66e00fd91479953227fd510c29 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 5 Sep 2022 21:48:31 +0900
Subject: [PATCH 2127/3199] save

---
 runtime/src/bank.rs | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 107da41ff03eff..2129e15ff88c7d 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -6388,7 +6388,8 @@ impl Bank {
             } else {
                 r
             }
-        }
+        };
+
         for (st, &i) in batch.sanitized_transactions().iter().zip(transaction_indexes.iter()) {
             scheduler.schedule(st, i);
         }

From c5f7b6773212a746bae475b9694a3f70aa4e1195 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 5 Sep 2022 21:48:47 +0900
Subject: [PATCH 2128/3199] save

---
 runtime/src/bank.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 2129e15ff88c7d..602c6afb628dbb 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -6382,7 +6382,7 @@ impl Bank {
         let scheduler = {
             let r = self.scheduler.read().unwrap();
 
-            if scheduler.bank.read().unwrap().is_none() {
+            if r.bank.read().unwrap().is_none() {
                 drop(r);
                 panic!();
             } else {

From d5538dd9d9e0b7809da1f6280e0a50377e57d734 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 5 Sep 2022 21:50:14 +0900
Subject: [PATCH 2129/3199] save

---
 runtime/src/bank.rs | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 602c6afb628dbb..08806c7e4dd74f 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -6384,6 +6384,8 @@ impl Bank {
 
             if r.bank.read().unwrap().is_none() {
                 drop(r);
+                let w = self.scheduler.write().unwrap();
+                w.bank.write().unwrap() = Some(Arc::clone(batch.bank));
                 panic!();
             } else {
                 r

From 503feb4bb4ef1a9ef6a5d92447ac1c9b3445b654 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 5 Sep 2022 21:50:31 +0900
Subject: [PATCH 2130/3199] save

---
 runtime/src/bank.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 08806c7e4dd74f..9fccfc19af593b 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -6385,7 +6385,7 @@ impl Bank {
             if r.bank.read().unwrap().is_none() {
                 drop(r);
                 let w = self.scheduler.write().unwrap();
-                w.bank.write().unwrap() = Some(Arc::clone(batch.bank));
+                w.bank.write().unwrap() = Some(Arc::clone(batch.bank()));
                 panic!();
             } else {
                 r

From d92142affc46808ae7250aa7012a498b2b8cbfc2 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 5 Sep 2022 21:52:01 +0900
Subject: [PATCH 2131/3199] save

---
 runtime/src/bank.rs | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 9fccfc19af593b..f2a4b875ef8870 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -6377,6 +6377,7 @@ impl Bank {
     pub fn schedule_and_commit_transactions(
         &self,
         batch: &TransactionBatch,
+        bank, &Arc<Bank>,
         transaction_indexes: &Vec<usize>,
     ) {
         let scheduler = {
@@ -6385,7 +6386,7 @@ impl Bank {
             if r.bank.read().unwrap().is_none() {
                 drop(r);
                 let w = self.scheduler.write().unwrap();
-                w.bank.write().unwrap() = Some(Arc::clone(batch.bank()));
+                w.bank.write().unwrap() = Some(Arc::clone(bank));
                 panic!();
             } else {
                 r

From 1387a2b56fe286443cbc7d07b6b403ed3188f555 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 5 Sep 2022 21:52:11 +0900
Subject: [PATCH 2132/3199] save

---
 runtime/src/bank.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index f2a4b875ef8870..8dad4270a13197 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -6377,7 +6377,7 @@ impl Bank {
     pub fn schedule_and_commit_transactions(
         &self,
         batch: &TransactionBatch,
-        bank, &Arc<Bank>,
+        bank: &Arc<Bank>,
         transaction_indexes: &Vec<usize>,
     ) {
         let scheduler = {

From e469875ed7a0b410531be2cd8396af2881701836 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 5 Sep 2022 21:52:26 +0900
Subject: [PATCH 2133/3199] save

---
 runtime/src/bank.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 8dad4270a13197..78e4844970537b 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -6386,7 +6386,7 @@ impl Bank {
             if r.bank.read().unwrap().is_none() {
                 drop(r);
                 let w = self.scheduler.write().unwrap();
-                w.bank.write().unwrap() = Some(Arc::clone(bank));
+                *w.bank.write().unwrap() = Some(Arc::clone(bank));
                 panic!();
             } else {
                 r

From 6605caeb07006535ff390905f582a5b1761b3406 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 5 Sep 2022 21:53:33 +0900
Subject: [PATCH 2134/3199] save

---
 runtime/src/bank.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 78e4844970537b..49191c457355ba 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1205,7 +1205,7 @@ struct Scheduler {
     preloader: Arc<solana_scheduler::Preloader>,
     graceful_stop_initiated: bool,
     errors: Arc<std::sync::Mutex<Vec<Result<()>>>>,
-    bank: std::sync::Arc<std::sync::RwLock<std::option::Option<std::option::Option<std::sync::Arc<Bank>>>>>,
+    bank: std::sync::Arc<std::sync::RwLock<std::option::Option<std::sync::Arc<Bank>>>>,
 }
 
 impl Scheduler {

From 78105744e5e2a8aae2b615db1911cc3c44bcaa78 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 5 Sep 2022 21:54:23 +0900
Subject: [PATCH 2135/3199] save

---
 runtime/src/bank.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 49191c457355ba..9c9b38f3c511e3 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1241,7 +1241,7 @@ impl Default for Scheduler {
         let (completed_ee_sender, completed_ee_receiver) = crossbeam_channel::unbounded();
         let (droppable_ee_sender, droppable_ee_receiver) = crossbeam_channel::unbounded();
 
-        let bank = Arc::new(std::sync::RwLock::new(None::<Option<Arc<Bank>>>));
+        let bank = Arc::new(std::sync::RwLock::new(None::<Arc<Bank>>));
 
         let executing_thread_handles = (0..8).map(|thx| {
             let (scheduled_ee_receiver, completed_ee_sender) = (scheduled_ee_receiver.clone(), completed_ee_sender.clone());

From decb969156d2c3a27c5ff81016f2e5301c157480 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 5 Sep 2022 21:54:53 +0900
Subject: [PATCH 2136/3199] save

---
 runtime/src/bank.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 9c9b38f3c511e3..68e4aed0a30ddd 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1251,7 +1251,7 @@ impl Default for Scheduler {
             while let Ok(solana_scheduler::ExecutablePayload(mut ee)) = scheduled_ee_receiver.recv() {
                 let bank_r = bank.read().unwrap();
                 let bank_o = (&bank_r).as_ref().unwrap();
-                let bank = bank_o.as_ref().unwrap();
+                let bank = bank_o.as_ref();
 
                 let tx_account_lock_limit = bank.get_transaction_account_lock_limit();
                 let lock_result = ee.task.tx.0

From 0de9bf8ad964530b94574d5404e17297385565bf Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 5 Sep 2022 21:55:19 +0900
Subject: [PATCH 2137/3199] save

---
 ledger/src/blockstore_processor.rs | 1 +
 1 file changed, 1 insertion(+)

diff --git a/ledger/src/blockstore_processor.rs b/ledger/src/blockstore_processor.rs
index a5306bc32c6d0e..8cf7e6286156dd 100644
--- a/ledger/src/blockstore_processor.rs
+++ b/ledger/src/blockstore_processor.rs
@@ -192,6 +192,7 @@ fn execute_batch(
 
     batch.bank().schedule_and_commit_transactions(
         batch,
+        bank,
         transaction_indexes,
     );
     if let Some(first_error_from_scheduler) = batch.bank().handle_aborted_transactions().into_iter().next() {

From 4d036bf77b6f1c66c83f7418a189111e4ff19b5e Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 5 Sep 2022 22:01:43 +0900
Subject: [PATCH 2138/3199] save

---
 runtime/src/bank.rs | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 68e4aed0a30ddd..74d0dc13e04a4f 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -8055,8 +8055,7 @@ impl Bank {
     pub fn wait_for_scheduler(&self) -> Result<()> {
         let mut scheduler = self.scheduler.write().unwrap();
         scheduler.gracefully_stop()?;
-
-        Ok(())
+        scheduler.handle_aborted_executions().get(0).or(Ok(()))
     }
 }
 

From 03c2f19c7e0875b61bbb7970cf1696774bccaccc Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 5 Sep 2022 22:02:44 +0900
Subject: [PATCH 2139/3199] save

---
 runtime/src/bank.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 74d0dc13e04a4f..6d52dd7052391a 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -8055,7 +8055,7 @@ impl Bank {
     pub fn wait_for_scheduler(&self) -> Result<()> {
         let mut scheduler = self.scheduler.write().unwrap();
         scheduler.gracefully_stop()?;
-        scheduler.handle_aborted_executions().get(0).or(Ok(()))
+        scheduler.handle_aborted_executions().get(0).or_else(Ok(()))
     }
 }
 

From 7618a1ebf1c4db84aafdb6d095856e5e0dcb8b41 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 5 Sep 2022 22:03:00 +0900
Subject: [PATCH 2140/3199] save

---
 runtime/src/bank.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 6d52dd7052391a..df4b5696b5e47b 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -8055,7 +8055,7 @@ impl Bank {
     pub fn wait_for_scheduler(&self) -> Result<()> {
         let mut scheduler = self.scheduler.write().unwrap();
         scheduler.gracefully_stop()?;
-        scheduler.handle_aborted_executions().get(0).or_else(Ok(()))
+        scheduler.handle_aborted_executions().get(0).or_else(|| Ok(()))
     }
 }
 

From bf470393d0ca3135ad4d9985e458632f5be44607 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 5 Sep 2022 22:03:30 +0900
Subject: [PATCH 2141/3199] save

---
 runtime/src/bank.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index df4b5696b5e47b..7fa812e8610fc9 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -8055,7 +8055,7 @@ impl Bank {
     pub fn wait_for_scheduler(&self) -> Result<()> {
         let mut scheduler = self.scheduler.write().unwrap();
         scheduler.gracefully_stop()?;
-        scheduler.handle_aborted_executions().get(0).or_else(|| Ok(()))
+        scheduler.handle_aborted_executions().get(0).unwrap_or(|| Ok(()))
     }
 }
 

From 4b50b8318597fa7c3b1de468967a294929a1846f Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 5 Sep 2022 22:03:47 +0900
Subject: [PATCH 2142/3199] save

---
 runtime/src/bank.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 7fa812e8610fc9..31772081c77cd1 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -8055,7 +8055,7 @@ impl Bank {
     pub fn wait_for_scheduler(&self) -> Result<()> {
         let mut scheduler = self.scheduler.write().unwrap();
         scheduler.gracefully_stop()?;
-        scheduler.handle_aborted_executions().get(0).unwrap_or(|| Ok(()))
+        scheduler.handle_aborted_executions().get(0).unwrap_or(Ok(()))
     }
 }
 

From 1a7b8bc683525425f2eb784fcfaa0ff10e16323d Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 5 Sep 2022 22:04:06 +0900
Subject: [PATCH 2143/3199] save

---
 runtime/src/bank.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 31772081c77cd1..773a6ac8814365 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -8055,7 +8055,7 @@ impl Bank {
     pub fn wait_for_scheduler(&self) -> Result<()> {
         let mut scheduler = self.scheduler.write().unwrap();
         scheduler.gracefully_stop()?;
-        scheduler.handle_aborted_executions().get(0).unwrap_or(Ok(()))
+        scheduler.handle_aborted_executions().get(0).unwrap_or(&Ok(()))
     }
 }
 

From 6902b255437efe6b573db2c21050a1ff0154857d Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 5 Sep 2022 22:04:20 +0900
Subject: [PATCH 2144/3199] save

---
 runtime/src/bank.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 773a6ac8814365..20a4adb19135c3 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -8055,7 +8055,7 @@ impl Bank {
     pub fn wait_for_scheduler(&self) -> Result<()> {
         let mut scheduler = self.scheduler.write().unwrap();
         scheduler.gracefully_stop()?;
-        scheduler.handle_aborted_executions().get(0).unwrap_or(&Ok(()))
+        scheduler.handle_aborted_executions().get(0).cloned().unwrap_or(Ok(()))
     }
 }
 

From 9a607d61ab44b2458dfe3547be64a2b8edb09d43 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 5 Sep 2022 22:06:06 +0900
Subject: [PATCH 2145/3199] save

---
 runtime/src/bank.rs  | 4 ++--
 scheduler/src/lib.rs | 4 ++++
 2 files changed, 6 insertions(+), 2 deletions(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 20a4adb19135c3..bac5beb4429cf5 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1290,8 +1290,8 @@ impl Default for Scheduler {
 
         let error_collector_thread_handle = std::thread::Builder::new().name(format!("solErrorCol{:02}", 0)).spawn(move || {
             while let Ok(solana_scheduler::DroppablePayload(_ee)) = droppable_ee_receiver.recv() {
-                if false /* ee.is_aborted() */ {
-                    errors_in_collector_thread.lock().unwrap().push(Ok(()))
+                if ee.is_aborted() {
+                    errors_in_collector_thread.lock().unwrap().push(ee.execution_result.clone());
                 }
             }
             Ok(())
diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 180cb1ab02d5dc..7eb5285522b308 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -108,6 +108,10 @@ impl ExecutionEnvironment {
     fn is_reindexed(&self) -> bool {
         self.is_reindexed
     }
+
+    fn is_aborted(&self) -> bool {
+        self.execution_result.is_err()
+    }
 }
 
 unsafe trait AtScheduleThread: Copy {}

From 0958454b43b37d6ee4c0cde2a9d1d9d5356e1894 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 5 Sep 2022 22:07:07 +0900
Subject: [PATCH 2146/3199] save

---
 scheduler/src/lib.rs | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 7eb5285522b308..a566b5d7096777 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -110,7 +110,11 @@ impl ExecutionEnvironment {
     }
 
     fn is_aborted(&self) -> bool {
-        self.execution_result.is_err()
+        if let Some(r) = self.execution_result {
+            r.is_err()
+        } else {
+            false
+        }
     }
 }
 

From 15693c873f82296d2b44d91617caa352c4e15176 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 5 Sep 2022 22:07:17 +0900
Subject: [PATCH 2147/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index a566b5d7096777..4af54bfcaca3d6 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -110,7 +110,7 @@ impl ExecutionEnvironment {
     }
 
     fn is_aborted(&self) -> bool {
-        if let Some(r) = self.execution_result {
+        if let Some(r) = &self.execution_result {
             r.is_err()
         } else {
             false

From f27113f2fc9434e4a58bf4ea4916158c9af9530c Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 5 Sep 2022 22:07:31 +0900
Subject: [PATCH 2148/3199] save

---
 runtime/src/bank.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index bac5beb4429cf5..6128253694e30e 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1289,7 +1289,7 @@ impl Default for Scheduler {
         let errors_in_collector_thread = Arc::clone(&errors);
 
         let error_collector_thread_handle = std::thread::Builder::new().name(format!("solErrorCol{:02}", 0)).spawn(move || {
-            while let Ok(solana_scheduler::DroppablePayload(_ee)) = droppable_ee_receiver.recv() {
+            while let Ok(solana_scheduler::DroppablePayload(ee)) = droppable_ee_receiver.recv() {
                 if ee.is_aborted() {
                     errors_in_collector_thread.lock().unwrap().push(ee.execution_result.clone());
                 }

From 65d6e4bc8ba779b1b7cac77a4bcc1a17f1c3f844 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 5 Sep 2022 22:08:08 +0900
Subject: [PATCH 2149/3199] save

---
 runtime/src/bank.rs  | 2 +-
 scheduler/src/lib.rs | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 6128253694e30e..ce10b3cdb7d776 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1291,7 +1291,7 @@ impl Default for Scheduler {
         let error_collector_thread_handle = std::thread::Builder::new().name(format!("solErrorCol{:02}", 0)).spawn(move || {
             while let Ok(solana_scheduler::DroppablePayload(ee)) = droppable_ee_receiver.recv() {
                 if ee.is_aborted() {
-                    errors_in_collector_thread.lock().unwrap().push(ee.execution_result.clone());
+                    errors_in_collector_thread.lock().unwrap().push(ee.execution_result.unwrap().clone());
                 }
             }
             Ok(())
diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 4af54bfcaca3d6..61aa36f671e134 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -109,7 +109,7 @@ impl ExecutionEnvironment {
         self.is_reindexed
     }
 
-    fn is_aborted(&self) -> bool {
+    pub fn is_aborted(&self) -> bool {
         if let Some(r) = &self.execution_result {
             r.is_err()
         } else {

From c5d9e05f21632917c694eae544f257c188d6c4f3 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 5 Sep 2022 22:08:39 +0900
Subject: [PATCH 2150/3199] save

---
 runtime/src/bank.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index ce10b3cdb7d776..88bd81c985f58a 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1291,7 +1291,7 @@ impl Default for Scheduler {
         let error_collector_thread_handle = std::thread::Builder::new().name(format!("solErrorCol{:02}", 0)).spawn(move || {
             while let Ok(solana_scheduler::DroppablePayload(ee)) = droppable_ee_receiver.recv() {
                 if ee.is_aborted() {
-                    errors_in_collector_thread.lock().unwrap().push(ee.execution_result.unwrap().clone());
+                    errors_in_collector_thread.lock().unwrap().push(ee.execution_result.take().unwrap());
                 }
             }
             Ok(())

From 2d61d2aa42aeaf1484b1c9d9eaea47b7577bfba5 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 5 Sep 2022 22:08:56 +0900
Subject: [PATCH 2151/3199] save

---
 runtime/src/bank.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 88bd81c985f58a..f3631c9f0330ab 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1289,7 +1289,7 @@ impl Default for Scheduler {
         let errors_in_collector_thread = Arc::clone(&errors);
 
         let error_collector_thread_handle = std::thread::Builder::new().name(format!("solErrorCol{:02}", 0)).spawn(move || {
-            while let Ok(solana_scheduler::DroppablePayload(ee)) = droppable_ee_receiver.recv() {
+            while let Ok(solana_scheduler::DroppablePayload(mut ee)) = droppable_ee_receiver.recv() {
                 if ee.is_aborted() {
                     errors_in_collector_thread.lock().unwrap().push(ee.execution_result.take().unwrap());
                 }

From 73a169e4f208067f9e542e1669077f8ee927b24f Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 5 Sep 2022 22:10:57 +0900
Subject: [PATCH 2152/3199] save

---
 runtime/src/bank.rs | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index f3631c9f0330ab..7f63ffd005ea04 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1291,6 +1291,10 @@ impl Default for Scheduler {
         let error_collector_thread_handle = std::thread::Builder::new().name(format!("solErrorCol{:02}", 0)).spawn(move || {
             while let Ok(solana_scheduler::DroppablePayload(mut ee)) = droppable_ee_receiver.recv() {
                 if ee.is_aborted() {
+                    warn!(
+                        "scheduler: Unexpected validator error: {:?}, transaction: {:?}",
+                        ee.execution_result, ee.task.tx.0
+                    );
                     errors_in_collector_thread.lock().unwrap().push(ee.execution_result.take().unwrap());
                 }
             }

From f88b2a22ef9324efeedd07e1bc3191850dae64e5 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 5 Sep 2022 22:11:37 +0900
Subject: [PATCH 2153/3199] save

---
 runtime/src/bank.rs | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 7f63ffd005ea04..0c27d9873d88ab 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1297,6 +1297,8 @@ impl Default for Scheduler {
                     );
                     errors_in_collector_thread.lock().unwrap().push(ee.execution_result.take().unwrap());
                 }
+
+                drop(ee);
             }
             Ok(())
         }).unwrap();

From b00be32485938c594ad0e3a083b55c3bf412f403 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 5 Sep 2022 22:14:01 +0900
Subject: [PATCH 2154/3199] save

---
 runtime/src/bank.rs | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 0c27d9873d88ab..cf5ab10211343a 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -6393,7 +6393,8 @@ impl Bank {
                 drop(r);
                 let w = self.scheduler.write().unwrap();
                 *w.bank.write().unwrap() = Some(Arc::clone(bank));
-                panic!();
+                drop(w);
+                self.scheduler.read().unwrap()
             } else {
                 r
             }

From 2a77f22f84f0af7d0184ba118cea9721b80ae964 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 5 Sep 2022 22:24:08 +0900
Subject: [PATCH 2155/3199] save

---
 ledger/src/blockstore_processor.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ledger/src/blockstore_processor.rs b/ledger/src/blockstore_processor.rs
index 8cf7e6286156dd..d6ec3c3d171251 100644
--- a/ledger/src/blockstore_processor.rs
+++ b/ledger/src/blockstore_processor.rs
@@ -1675,7 +1675,7 @@ fn process_single_slot(
         err
     })?;
 
-    bank.wait_for_scheduler().unwrap();
+    bank.wait_for_scheduler()?
     bank.freeze(); // all banks handled by this routine are created from complete slots
     if blockstore.is_primary_access() {
         blockstore.insert_bank_hash(bank.slot(), bank.hash(), false);

From 0ece917d6103b160a074a45b4447e8fdc604d3f5 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 5 Sep 2022 22:24:35 +0900
Subject: [PATCH 2156/3199] save

---
 ledger/src/blockstore_processor.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ledger/src/blockstore_processor.rs b/ledger/src/blockstore_processor.rs
index d6ec3c3d171251..b25b382fb6f926 100644
--- a/ledger/src/blockstore_processor.rs
+++ b/ledger/src/blockstore_processor.rs
@@ -1675,7 +1675,7 @@ fn process_single_slot(
         err
     })?;
 
-    bank.wait_for_scheduler()?
+    bank.wait_for_scheduler()?;
     bank.freeze(); // all banks handled by this routine are created from complete slots
     if blockstore.is_primary_access() {
         blockstore.insert_bank_hash(bank.slot(), bank.hash(), false);

From f49adf0a69bf62a806219778968cc2d812a93228 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 5 Sep 2022 22:25:18 +0900
Subject: [PATCH 2157/3199] save

---
 ledger/src/blockstore_processor.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ledger/src/blockstore_processor.rs b/ledger/src/blockstore_processor.rs
index b25b382fb6f926..4431c2fcc718dc 100644
--- a/ledger/src/blockstore_processor.rs
+++ b/ledger/src/blockstore_processor.rs
@@ -1326,7 +1326,7 @@ fn process_bank_0(
         &mut ExecuteTimings::default(),
     )
     .expect("Failed to process bank 0 from ledger. Did you forget to provide a snapshot?");
-    bank0.wait_for_scheduler().unwrap();
+    bank0.wait_for_scheduler()?;
     bank0.freeze();
     if blockstore.is_primary_access() {
         blockstore.insert_bank_hash(bank0.slot(), bank0.hash(), false);

From 212d1382f7fcae39c383ff886d003d9757f57225 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 5 Sep 2022 22:27:20 +0900
Subject: [PATCH 2158/3199] save

---
 ledger/src/blockstore_processor.rs | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/ledger/src/blockstore_processor.rs b/ledger/src/blockstore_processor.rs
index 4431c2fcc718dc..0d3cd75772bb37 100644
--- a/ledger/src/blockstore_processor.rs
+++ b/ledger/src/blockstore_processor.rs
@@ -198,7 +198,9 @@ fn execute_batch(
     if let Some(first_error_from_scheduler) = batch.bank().handle_aborted_transactions().into_iter().next() {
         first_error_from_scheduler?
     }
+    return Ok(());
 
+    /*
     let (tx_results, balances) = batch.bank().load_execute_and_commit_transactions(
         batch,
         MAX_PROCESSING_AGE,
@@ -271,6 +273,7 @@ fn execute_batch(
 
     let first_err = get_first_error(batch, fee_collection_results);
     first_err.map(|(result, _)| result).unwrap_or(Ok(()))
+    */
 }
 
 #[derive(Default)]

From 2c35ca1fb61f13aa77d9bf6eec52d5243dd46a78 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 5 Sep 2022 22:28:40 +0900
Subject: [PATCH 2159/3199] save

---
 ledger/src/blockstore_processor.rs | 2 +-
 runtime/src/bank.rs                | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/ledger/src/blockstore_processor.rs b/ledger/src/blockstore_processor.rs
index 0d3cd75772bb37..97ab0dc1ecc553 100644
--- a/ledger/src/blockstore_processor.rs
+++ b/ledger/src/blockstore_processor.rs
@@ -1329,7 +1329,7 @@ fn process_bank_0(
         &mut ExecuteTimings::default(),
     )
     .expect("Failed to process bank 0 from ledger. Did you forget to provide a snapshot?");
-    bank0.wait_for_scheduler()?;
+    bank0.wait_for_scheduler().unwrap();
     bank0.freeze();
     if blockstore.is_primary_access() {
         blockstore.insert_bank_hash(bank0.slot(), bank0.hash(), false);
diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index cf5ab10211343a..495e5b300053a1 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -8062,7 +8062,7 @@ impl Bank {
     pub fn wait_for_scheduler(&self) -> Result<()> {
         let mut scheduler = self.scheduler.write().unwrap();
         scheduler.gracefully_stop()?;
-        scheduler.handle_aborted_executions().get(0).cloned().unwrap_or(Ok(()))
+        scheduler.handle_aborted_executions().into_iter().next().unwrap_or(Ok(()))
     }
 }
 

From cdfc7fea556f4f59fc689beaa818e28510fdbc18 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 5 Sep 2022 22:46:49 +0900
Subject: [PATCH 2160/3199] save

---
 runtime/src/bank.rs  |  2 +-
 scheduler/src/lib.rs | 16 ++++++++--------
 2 files changed, 9 insertions(+), 9 deletions(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 495e5b300053a1..58a465fd3c7d2c 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1279,7 +1279,7 @@ impl Default for Scheduler {
                 ee.execution_result = Some(fee_collection_results.into_iter().collect::<Result<_>>());
 
 
-                ee.reindex_with_address_book();
+                //ee.reindex_with_address_book();
                 completed_ee_sender.send(solana_scheduler::UnlockablePayload(ee)).unwrap();
             }
             Ok(())
diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 61aa36f671e134..eda654ad682227 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -711,13 +711,13 @@ impl Task {
         this: &TaskInQueue,
         task_sender: &crossbeam_channel::Sender<(TaskInQueue, Vec<LockAttempt>)>,
     ) {
-        //for lock_attempt in self.lock_attempts_mut(ast).iter() {
-        //    lock_attempt.contended_unique_weights().insert_task(unique_weight, Task::clone_in_queue(&self));
-        //}
-        let a = Task::clone_in_queue(this);
-        task_sender
-            .send((a, std::mem::take(&mut *this.for_indexer.0.borrow_mut())))
-            .unwrap();
+        for lock_attempt in self.lock_attempts_mut(ast).iter() {
+            lock_attempt.contended_unique_weights().insert_task(unique_weight, Task::clone_in_queue(&self));
+        }
+        //let a = Task::clone_in_queue(this);
+        //task_sender
+        //    .send((a, std::mem::take(&mut *this.for_indexer.0.borrow_mut())))
+        //    .unwrap();
     }
 
     fn stuck_task_id(&self) -> StuckTaskId {
@@ -1236,7 +1236,7 @@ impl ScheduleStage {
     ) {
         // do par()-ly?
 
-        //ee.reindex_with_address_book();
+        ee.reindex_with_address_book();
         assert!(ee.is_reindexed());
 
         ee.task.record_commit_time(*commit_time);

From 684add3eff896b8c1c040b647204964d7441e91f Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 5 Sep 2022 22:47:09 +0900
Subject: [PATCH 2161/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index eda654ad682227..1124b4f16ea900 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -979,7 +979,7 @@ impl ScheduleStage {
                         next_task.mark_as_contended();
                         *contended_count = contended_count.checked_add(1).unwrap();
 
-                        Task::index_with_address_book(&next_task, task_sender);
+                        Task::index_with_address_book(ast, &next_task, task_sender);
 
                         // maybe run lightweight prune logic on contended_queue here.
                     } else {

From 3834185d3e57ae84590f51adc63dbea1d91ac017 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 5 Sep 2022 22:47:42 +0900
Subject: [PATCH 2162/3199] save

---
 scheduler/src/lib.rs | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 1124b4f16ea900..820de0acca8511 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -707,7 +707,8 @@ impl Task {
     }
 
     #[inline(never)]
-    fn index_with_address_book(
+    fn index_with_address_book<AST: AtScheduleThread>(
+        ast: AST,
         this: &TaskInQueue,
         task_sender: &crossbeam_channel::Sender<(TaskInQueue, Vec<LockAttempt>)>,
     ) {

From 29c4c46a321f0cb1bff0e9965350258a91900249 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 5 Sep 2022 22:48:10 +0900
Subject: [PATCH 2163/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 820de0acca8511..f068b9407abdf1 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -712,7 +712,7 @@ impl Task {
         this: &TaskInQueue,
         task_sender: &crossbeam_channel::Sender<(TaskInQueue, Vec<LockAttempt>)>,
     ) {
-        for lock_attempt in self.lock_attempts_mut(ast).iter() {
+        for lock_attempt in this.lock_attempts_mut(ast).iter() {
             lock_attempt.contended_unique_weights().insert_task(unique_weight, Task::clone_in_queue(&self));
         }
         //let a = Task::clone_in_queue(this);

From b7003d41aad80defbac555c0abcc78d631c797ca Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 5 Sep 2022 22:48:22 +0900
Subject: [PATCH 2164/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index f068b9407abdf1..87b975da2a5ba5 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -713,7 +713,7 @@ impl Task {
         task_sender: &crossbeam_channel::Sender<(TaskInQueue, Vec<LockAttempt>)>,
     ) {
         for lock_attempt in this.lock_attempts_mut(ast).iter() {
-            lock_attempt.contended_unique_weights().insert_task(unique_weight, Task::clone_in_queue(&self));
+            lock_attempt.contended_unique_weights().insert_task(unique_weight, Task::clone_in_queue(&this));
         }
         //let a = Task::clone_in_queue(this);
         //task_sender

From 7727985df7141bee78824bfabf583bc7379d87ff Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 5 Sep 2022 22:48:29 +0900
Subject: [PATCH 2165/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 87b975da2a5ba5..5aea0178cd86a7 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -713,7 +713,7 @@ impl Task {
         task_sender: &crossbeam_channel::Sender<(TaskInQueue, Vec<LockAttempt>)>,
     ) {
         for lock_attempt in this.lock_attempts_mut(ast).iter() {
-            lock_attempt.contended_unique_weights().insert_task(unique_weight, Task::clone_in_queue(&this));
+            lock_attempt.contended_unique_weights().insert_task(unique_weight, Task::clone_in_queue(this));
         }
         //let a = Task::clone_in_queue(this);
         //task_sender

From a67838db6783f7d4e51f43365730d28cbd4a40c8 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 5 Sep 2022 22:48:54 +0900
Subject: [PATCH 2166/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 5aea0178cd86a7..e217f4c328f334 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -713,7 +713,7 @@ impl Task {
         task_sender: &crossbeam_channel::Sender<(TaskInQueue, Vec<LockAttempt>)>,
     ) {
         for lock_attempt in this.lock_attempts_mut(ast).iter() {
-            lock_attempt.contended_unique_weights().insert_task(unique_weight, Task::clone_in_queue(this));
+            lock_attempt.contended_unique_weights().insert_task(this.unique_weight, Task::clone_in_queue(this));
         }
         //let a = Task::clone_in_queue(this);
         //task_sender

From 044828570de4eca4a69fc958460a6f9faac7ec32 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 5 Sep 2022 23:05:23 +0900
Subject: [PATCH 2167/3199] save

---
 scheduler/src/lib.rs | 11 ++++++++---
 1 file changed, 8 insertions(+), 3 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index e217f4c328f334..30a71f07e40566 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -885,7 +885,7 @@ impl ScheduleStage {
             }
             (None, None) => {
                 trace!("select: none");
-                if runnable_queue.task_count() == 0 && /* *contended_count > 0 &&*/ address_book.stuck_tasks.len() > 0
+                if false && runnable_queue.task_count() == 0 && /* *contended_count > 0 &&*/ address_book.stuck_tasks.len() > 0
                 {
                     trace!("handling stuck...");
                     let (stuck_task_id, task) = address_book.stuck_tasks.pop_first().unwrap();
@@ -1002,10 +1002,13 @@ impl ScheduleStage {
                         // stuck_tasks, so pretend to add anew one.
                         // todo: optimize this needless operation
                         next_task.update_busiest_page_cu(busiest_page_cu);
+                        /*
                         let a = address_book
                             .stuck_tasks
                             .insert(next_task.stuck_task_id(), Task::clone_in_queue(&next_task));
                         assert!(a.is_none());
+                        */
+
                         if from_runnable {
                             continue; // continue to prefer depleting the possibly-non-empty runnable queue
                         } else if task_source == TaskSource::Stuck {
@@ -1020,6 +1023,7 @@ impl ScheduleStage {
                         }
                     } else {
                         // todo: remove this task from stuck_tasks before update_busiest_page_cu
+                        /*
                         let removed = address_book
                             .stuck_tasks
                             .remove(&next_task.stuck_task_id())
@@ -1029,6 +1033,7 @@ impl ScheduleStage {
                             .stuck_tasks
                             .insert(next_task.stuck_task_id(), removed);
                         assert!(a.is_none());
+                        */
                         break;
                     }
                 } else if provisional_count > 0 {
@@ -1038,7 +1043,7 @@ impl ScheduleStage {
                     trace!("provisional exec: [{}/{}]", provisional_count, lock_count);
                     *contended_count = contended_count.checked_sub(1).unwrap();
                     next_task.mark_as_uncontended();
-                    address_book.stuck_tasks.remove(&next_task.stuck_task_id());
+                    //address_book.stuck_tasks.remove(&next_task.stuck_task_id());
                     next_task.update_busiest_page_cu(busiest_page_cu);
 
                     let tracker = triomphe::Arc::new(ProvisioningTracker::new(
@@ -1254,7 +1259,7 @@ impl ScheduleStage {
         );
         ee.task.mark_as_finished();
 
-        address_book.stuck_tasks.remove(&ee.task.stuck_task_id());
+        //address_book.stuck_tasks.remove(&ee.task.stuck_task_id());
 
         // block-wide qos validation will be done here
         // if error risen..:

From 74bdcbec2044b07cc259cbb0803b8df8b6eede46 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 6 Sep 2022 10:34:10 +0900
Subject: [PATCH 2168/3199] save

---
 ledger-tool/Cargo.toml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/ledger-tool/Cargo.toml b/ledger-tool/Cargo.toml
index 5c17ff39818e35..ae45d4b922d0cd 100644
--- a/ledger-tool/Cargo.toml
+++ b/ledger-tool/Cargo.toml
@@ -30,6 +30,7 @@ solana-geyser-plugin-manager = { path = "../geyser-plugin-manager", version = "=
 solana-ledger = { path = "../ledger", version = "=1.12.0" }
 solana-logger = { path = "../logger", version = "=1.12.0" }
 solana-measure = { path = "../measure", version = "=1.12.0" }
+solana-metrics = { path = "../metrics", version = "=1.12.0" }
 solana-runtime = { path = "../runtime", version = "=1.12.0" }
 solana-scheduler = { path = "../scheduler", version = "=1.12.0" }
 solana-sdk = { path = "../sdk", version = "=1.12.0" }

From 169702c160f9c026999308b1e579895b009445f0 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 6 Sep 2022 10:34:35 +0900
Subject: [PATCH 2169/3199] save

---
 Cargo.lock | 1 +
 1 file changed, 1 insertion(+)

diff --git a/Cargo.lock b/Cargo.lock
index 77d1e2af47e007..5c03faca5f4bc1 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -5549,6 +5549,7 @@ dependencies = [
  "solana-ledger",
  "solana-logger 1.12.0",
  "solana-measure",
+ "solana-metrics",
  "solana-runtime",
  "solana-scheduler",
  "solana-sdk 1.12.0",

From 7eb0b3b9b7cc09b98d2d0daa61c942684c6d4c52 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 6 Sep 2022 10:35:29 +0900
Subject: [PATCH 2170/3199] save

---
 ledger-tool/src/main.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ledger-tool/src/main.rs b/ledger-tool/src/main.rs
index b0d5aa60420237..691c0eca9ca1d0 100644
--- a/ledger-tool/src/main.rs
+++ b/ledger-tool/src/main.rs
@@ -467,7 +467,7 @@ fn output_slot(
                                 //    lock_attempt.contended_unique_weights().insert_task(weight, solana_scheduler::TaskInQueue::clone(&t));
                                 //}
 
-                                muxed_sender.send(t).unwrap();
+                                muxed_sender.send(solana_scheduler::SchedulablePayload(t)).unwrap();
                                 depth.fetch_add(1, Ordering::Relaxed);
                                 weight -= 1;
                             }

From 8d884c7f05ecc773120a80f2dc66bb5a0580c15e Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 6 Sep 2022 10:36:29 +0900
Subject: [PATCH 2171/3199] save

---
 ledger-tool/src/main.rs | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/ledger-tool/src/main.rs b/ledger-tool/src/main.rs
index 691c0eca9ca1d0..0805e7b6c580d6 100644
--- a/ledger-tool/src/main.rs
+++ b/ledger-tool/src/main.rs
@@ -455,6 +455,11 @@ fn output_slot(
                 let t1 = std::thread::Builder::new()
                     .name(format!("sol-producer{}", thx))
                     .spawn(move || {
+                        #[derive(Clone, Copy, Debug)]
+                        struct NotAtTopOfScheduleThread;
+                        unsafe impl solana_scheduler::NotAtScheduleThread for NotAtTopOfScheduleThread {}
+                        let nast = NotAtTopOfScheduleThread;
+
                         for i in 0..loop_count {
                             error!("started!: {} {}", i, txes.len());
                             for tx in txes.iter().map(|t| (t.0.clone(), t.1.iter().map(|l| l.clone_for_test()).collect::<Vec<_>>())) {
@@ -462,7 +467,7 @@ fn output_slot(
                                     std::thread::sleep(std::time::Duration::from_micros(10));
                                 }
 
-                                let t = solana_scheduler::Task::new_for_queue(weight, tx);
+                                let t = solana_scheduler::Task::new_for_queue(nast, weight, tx);
                                 //for lock_attempt in t.tx.1.iter() {
                                 //    lock_attempt.contended_unique_weights().insert_task(weight, solana_scheduler::TaskInQueue::clone(&t));
                                 //}

From 62bc5c27aec8f1ac6e08d581a63feb10f0e85053 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 6 Sep 2022 10:36:43 +0900
Subject: [PATCH 2172/3199] save

---
 ledger-tool/src/main.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ledger-tool/src/main.rs b/ledger-tool/src/main.rs
index 0805e7b6c580d6..759a449bb30a96 100644
--- a/ledger-tool/src/main.rs
+++ b/ledger-tool/src/main.rs
@@ -403,7 +403,7 @@ fn output_slot(
             .name(format!("sol-consumer{}", thx))
             .spawn(move || {
                 loop {
-                    let ee = post_schedule_env_receiver.recv().unwrap();
+                    let ee = post_schedule_env_receiver.recv().unwrap().0;
                     d.fetch_sub(1, Ordering::Relaxed);
                     let step = step.fetch_add(1, Ordering::Relaxed);
                     trace!(

From 4708764864a95d1b4675083196e4bd945cc5d7d7 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 6 Sep 2022 10:36:54 +0900
Subject: [PATCH 2173/3199] save

---
 ledger-tool/src/main.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ledger-tool/src/main.rs b/ledger-tool/src/main.rs
index 759a449bb30a96..3a24f514b92c14 100644
--- a/ledger-tool/src/main.rs
+++ b/ledger-tool/src/main.rs
@@ -286,8 +286,8 @@ fn output_slot(
                 &mut runnable_queue,
                 &mut address_book,
                 &muxed_receiver,
-                &post_execute_env_receiver,
                 &pre_execute_env_sender,
+                &post_execute_env_receiver,
                 Some(&post_schedule_env_sender),
             );
         })

From d3b4ae427cbc7a18e3972ec55e5f55aabc6d5c1e Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 6 Sep 2022 10:37:26 +0900
Subject: [PATCH 2174/3199] save

---
 runtime/src/vote_parser.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/src/vote_parser.rs b/runtime/src/vote_parser.rs
index 660328db0ab9f5..c2ef5ecaea06b0 100644
--- a/runtime/src/vote_parser.rs
+++ b/runtime/src/vote_parser.rs
@@ -13,7 +13,7 @@ use {
 pub type ParsedVote = (Pubkey, VoteTransaction, Option<Hash>, Signature);
 
 // Used for filtering out votes from the transaction log collector
-pub(crate) fn is_simple_vote_transaction(transaction: &SanitizedTransaction) -> bool {
+pub fn is_simple_vote_transaction(transaction: &SanitizedTransaction) -> bool {
     if transaction.message().instructions().len() == 1 {
         let (program_pubkey, instruction) = transaction
             .message()

From 34d21925e246087c4d2b1d906650c940c90e8697 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 6 Sep 2022 10:38:01 +0900
Subject: [PATCH 2175/3199] save

---
 ledger-tool/src/main.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ledger-tool/src/main.rs b/ledger-tool/src/main.rs
index 3a24f514b92c14..312994478c0530 100644
--- a/ledger-tool/src/main.rs
+++ b/ledger-tool/src/main.rs
@@ -176,7 +176,7 @@ fn output_entry(
                 if !skip_voting
                     || !solana_runtime::vote_parser::is_simple_vote_transaction(&sanitized_tx)
                 {
-                    let locks = sanitized_tx.get_account_locks().unwrap();
+                    let locks = sanitized_tx.get_account_locks(usize::max_value()).unwrap();
                     let writable_lock_iter = locks
                         .writable
                         .iter()

From 03a5e55c6477355b943739ece6ff335bd0f6aea5 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 6 Sep 2022 10:38:20 +0900
Subject: [PATCH 2176/3199] save

---
 ledger-tool/src/main.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ledger-tool/src/main.rs b/ledger-tool/src/main.rs
index 312994478c0530..80992b7f35a8e2 100644
--- a/ledger-tool/src/main.rs
+++ b/ledger-tool/src/main.rs
@@ -307,7 +307,7 @@ fn output_slot(
                     let send_metrics = std::env::var("SEND_METRICS").is_ok();
 
                     for step in 0.. {
-                        let mut ee = pre_execute_env_receiver.recv().unwrap();
+                        let mut ee = pre_execute_env_receiver.recv().unwrap().0;
                         if step % 1966 == 0 {
                             error!("executing!: {} {}", step, pre_execute_env_receiver.len());
                         }

From 65c305313402f053557c715e0435c8ace7c80041 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 6 Sep 2022 10:38:41 +0900
Subject: [PATCH 2177/3199] save

---
 ledger-tool/src/main.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ledger-tool/src/main.rs b/ledger-tool/src/main.rs
index 80992b7f35a8e2..47afc0fecc7924 100644
--- a/ledger-tool/src/main.rs
+++ b/ledger-tool/src/main.rs
@@ -378,7 +378,7 @@ fn output_slot(
                                 ()
                             });
                         }
-                        post_execute_env_sender.send(ee).unwrap();
+                        post_execute_env_sender.send(solana_scheduler::UnlockablePayload(ee)).unwrap();
                     }
                 })
                 .unwrap();

From 14f998a56d56481c0e90aa44591c4bfcaef92a70 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 6 Sep 2022 10:38:54 +0900
Subject: [PATCH 2178/3199] save

---
 ledger-tool/src/main.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ledger-tool/src/main.rs b/ledger-tool/src/main.rs
index 47afc0fecc7924..54f0f21812b424 100644
--- a/ledger-tool/src/main.rs
+++ b/ledger-tool/src/main.rs
@@ -341,7 +341,7 @@ fn output_slot(
                             .unwrap();
                         */
                         let uq = ee.unique_weight;
-                        let should_remove = ee.task.contention_count > 0;
+                        let should_remove = ee.task.contention_count() > 0;
                         for mut lock_attempt in ee.lock_attempts.iter_mut() {
                             let contended_unique_weights = lock_attempt.contended_unique_weights();
                             contended_unique_weights.heaviest_task_cursor().map(|mut task_cursor| {

From d30eea65a2e0b9d3188ed435337741b58279a21b Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 6 Sep 2022 10:39:54 +0900
Subject: [PATCH 2179/3199] save

---
 scheduler/src/lib.rs | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 30a71f07e40566..9f4c4112378899 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -728,6 +728,11 @@ impl Task {
         assert_ne!(cu, 0);
         (cu, TaskId::max_value() - self.unique_weight)
     }
+
+    fn contention_count(&self) -> usize {
+        self.contention_count
+            .load(std::sync::atomic::Ordering::SeqCst)
+    }
 }
 
 // RunnableQueue, ContendedQueue?

From 9bb3c7f6f393bf2e76cbe71b6749aca3c43fe57e Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 6 Sep 2022 10:40:01 +0900
Subject: [PATCH 2180/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 9f4c4112378899..482ee46d6f6ece 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -729,7 +729,7 @@ impl Task {
         (cu, TaskId::max_value() - self.unique_weight)
     }
 
-    fn contention_count(&self) -> usize {
+    pub fn contention_count(&self) -> usize {
         self.contention_count
             .load(std::sync::atomic::Ordering::SeqCst)
     }

From 18a47a6d8c2385c8bd30e2f833eba6ea14de3710 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 6 Sep 2022 10:40:25 +0900
Subject: [PATCH 2181/3199] save

---
 ledger-tool/src/main.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ledger-tool/src/main.rs b/ledger-tool/src/main.rs
index 54f0f21812b424..3c2ae0568aec40 100644
--- a/ledger-tool/src/main.rs
+++ b/ledger-tool/src/main.rs
@@ -342,7 +342,7 @@ fn output_slot(
                         */
                         let uq = ee.unique_weight;
                         let should_remove = ee.task.contention_count() > 0;
-                        for mut lock_attempt in ee.lock_attempts.iter_mut() {
+                        for mut lock_attempt in ee.finalized_lock_attempts.iter_mut() {
                             let contended_unique_weights = lock_attempt.contended_unique_weights();
                             contended_unique_weights.heaviest_task_cursor().map(|mut task_cursor| {
                                 let mut found = true;

From f8a1ad3726b3f9b2bf73b9df1c027acfc65e6183 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 6 Sep 2022 12:30:42 +0900
Subject: [PATCH 2182/3199] save

---
 runtime/src/bank.rs | 1 +
 1 file changed, 1 insertion(+)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 58a465fd3c7d2c..8b3c5c3fbaac6b 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -4053,6 +4053,7 @@ impl Bank {
 
         inc_new_counter_debug!("bank-register_tick-registered", 1);
         if self.is_block_boundary(self.tick_height.load(Relaxed) + 1) {
+            self.wait_for_scheduler().unwrap();
             self.register_recent_blockhash(hash);
         }
 

From 8daf4284e508a929df0cbaec20d374213b1634e8 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 6 Sep 2022 12:59:18 +0900
Subject: [PATCH 2183/3199] save

---
 runtime/src/bank.rs | 18 ++++++++++--------
 1 file changed, 10 insertions(+), 8 deletions(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 8b3c5c3fbaac6b..c2836d6586d94c 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -4031,12 +4031,8 @@ impl Bank {
     /// reaches its max tick height. Can be called by tests to get new blockhashes for transaction
     /// processing without advancing to a new bank slot.
     pub fn register_recent_blockhash(&self, blockhash: &Hash) {
-        // Only acquire the write lock for the blockhash queue on block boundaries because
-        // readers can starve this write lock acquisition and ticks would be slowed down too
-        // much if the write lock is acquired for each tick.
-        let mut w_blockhash_queue = self.blockhash_queue.write().unwrap();
-        w_blockhash_queue.register_hash(blockhash, self.fee_rate_governor.lamports_per_signature);
-        self.update_recent_blockhashes_locked(&w_blockhash_queue);
+        let scheduler = self.scheduler.write().unwrap();
+        scheduler.blockhash = blockhash.clone();
     }
 
     /// Tell the bank which Entry IDs exist on the ledger. This function assumes subsequent calls
@@ -4053,7 +4049,6 @@ impl Bank {
 
         inc_new_counter_debug!("bank-register_tick-registered", 1);
         if self.is_block_boundary(self.tick_height.load(Relaxed) + 1) {
-            self.wait_for_scheduler().unwrap();
             self.register_recent_blockhash(hash);
         }
 
@@ -8063,7 +8058,14 @@ impl Bank {
     pub fn wait_for_scheduler(&self) -> Result<()> {
         let mut scheduler = self.scheduler.write().unwrap();
         scheduler.gracefully_stop()?;
-        scheduler.handle_aborted_executions().into_iter().next().unwrap_or(Ok(()))
+        scheduler.handle_aborted_executions().into_iter().next().unwrap_or(Ok(()))?
+
+        // Only acquire the write lock for the blockhash queue on block boundaries because
+        // readers can starve this write lock acquisition and ticks would be slowed down too
+        // much if the write lock is acquired for each tick.
+        let mut w_blockhash_queue = self.blockhash_queue.write().unwrap();
+        w_blockhash_queue.register_hash(scheduler.blockhash, self.fee_rate_governor.lamports_per_signature);
+        self.update_recent_blockhashes_locked(&w_blockhash_queue);
     }
 }
 

From d9b127894c8188ac1e0ac9dd498d7bfa9eb9954d Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 6 Sep 2022 12:59:29 +0900
Subject: [PATCH 2184/3199] save

---
 runtime/src/bank.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index c2836d6586d94c..68d6466a927088 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -8058,7 +8058,7 @@ impl Bank {
     pub fn wait_for_scheduler(&self) -> Result<()> {
         let mut scheduler = self.scheduler.write().unwrap();
         scheduler.gracefully_stop()?;
-        scheduler.handle_aborted_executions().into_iter().next().unwrap_or(Ok(()))?
+        scheduler.handle_aborted_executions().into_iter().next().unwrap_or(Ok(()))?;
 
         // Only acquire the write lock for the blockhash queue on block boundaries because
         // readers can starve this write lock acquisition and ticks would be slowed down too

From 2f7d1b3daa03153a42b25bcbec5d1968ca57f9a8 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 6 Sep 2022 12:59:49 +0900
Subject: [PATCH 2185/3199] save

---
 runtime/src/bank.rs | 1 +
 1 file changed, 1 insertion(+)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 68d6466a927088..12f13505b68cc1 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -8066,6 +8066,7 @@ impl Bank {
         let mut w_blockhash_queue = self.blockhash_queue.write().unwrap();
         w_blockhash_queue.register_hash(scheduler.blockhash, self.fee_rate_governor.lamports_per_signature);
         self.update_recent_blockhashes_locked(&w_blockhash_queue);
+        Ok(())
     }
 }
 

From b18f9e5b68917ead2c72a41e7d319dd63c7154b0 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 6 Sep 2022 13:00:31 +0900
Subject: [PATCH 2186/3199] save

---
 runtime/src/bank.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 12f13505b68cc1..ac6496629d3104 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -4032,7 +4032,7 @@ impl Bank {
     /// processing without advancing to a new bank slot.
     pub fn register_recent_blockhash(&self, blockhash: &Hash) {
         let scheduler = self.scheduler.write().unwrap();
-        scheduler.blockhash = blockhash.clone();
+        scheduler.blockhash = Some(blockhash.clone());
     }
 
     /// Tell the bank which Entry IDs exist on the ledger. This function assumes subsequent calls
@@ -8064,7 +8064,7 @@ impl Bank {
         // readers can starve this write lock acquisition and ticks would be slowed down too
         // much if the write lock is acquired for each tick.
         let mut w_blockhash_queue = self.blockhash_queue.write().unwrap();
-        w_blockhash_queue.register_hash(scheduler.blockhash, self.fee_rate_governor.lamports_per_signature);
+        w_blockhash_queue.register_hash(scheduler.blockhash.unwrap(), self.fee_rate_governor.lamports_per_signature);
         self.update_recent_blockhashes_locked(&w_blockhash_queue);
         Ok(())
     }

From 0abb73da5dc574477bd7be662ddd67731c555c69 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 6 Sep 2022 13:00:53 +0900
Subject: [PATCH 2187/3199] save

---
 runtime/src/bank.rs | 1 +
 1 file changed, 1 insertion(+)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index ac6496629d3104..c51c052fd0ea61 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1206,6 +1206,7 @@ struct Scheduler {
     graceful_stop_initiated: bool,
     errors: Arc<std::sync::Mutex<Vec<Result<()>>>>,
     bank: std::sync::Arc<std::sync::RwLock<std::option::Option<std::sync::Arc<Bank>>>>,
+    blockhash: Option<Hash>,
 }
 
 impl Scheduler {

From 4f367c8f4b636a487b44e71ca397656f564392fa Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 6 Sep 2022 13:01:21 +0900
Subject: [PATCH 2188/3199] save

---
 runtime/src/bank.rs | 1 +
 1 file changed, 1 insertion(+)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index c51c052fd0ea61..9746910f6f7876 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1332,6 +1332,7 @@ impl Default for Scheduler {
             graceful_stop_initiated: Default::default(),
             errors,
             bank,
+            blockhash: Default::default(),
         }
     }
 }

From e0426533cc9693b2929fb8cda96e957a706f03d6 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 6 Sep 2022 13:01:39 +0900
Subject: [PATCH 2189/3199] save

---
 runtime/src/bank.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 9746910f6f7876..2b8aa2bcb56a5b 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -8066,7 +8066,7 @@ impl Bank {
         // readers can starve this write lock acquisition and ticks would be slowed down too
         // much if the write lock is acquired for each tick.
         let mut w_blockhash_queue = self.blockhash_queue.write().unwrap();
-        w_blockhash_queue.register_hash(scheduler.blockhash.unwrap(), self.fee_rate_governor.lamports_per_signature);
+        w_blockhash_queue.register_hash(&scheduler.blockhash.unwrap(), self.fee_rate_governor.lamports_per_signature);
         self.update_recent_blockhashes_locked(&w_blockhash_queue);
         Ok(())
     }

From c30c9cfd52c930149103d77f60150d1de2084355 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 6 Sep 2022 13:02:01 +0900
Subject: [PATCH 2190/3199] save

---
 runtime/src/bank.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 2b8aa2bcb56a5b..53df8c9026e67e 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -4033,7 +4033,7 @@ impl Bank {
     /// reaches its max tick height. Can be called by tests to get new blockhashes for transaction
     /// processing without advancing to a new bank slot.
     pub fn register_recent_blockhash(&self, blockhash: &Hash) {
-        let scheduler = self.scheduler.write().unwrap();
+        let mut scheduler = self.scheduler.write().unwrap();
         scheduler.blockhash = Some(blockhash.clone());
     }
 

From b18c415c3f363450112558588c43c961f86d0b1d Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 6 Sep 2022 13:24:49 +0900
Subject: [PATCH 2191/3199] save

---
 runtime/src/bank.rs | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 53df8c9026e67e..e86c4339772df0 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1244,7 +1244,7 @@ impl Default for Scheduler {
 
         let bank = Arc::new(std::sync::RwLock::new(None::<Arc<Bank>>));
 
-        let executing_thread_handles = (0..8).map(|thx| {
+        let executing_thread_handles = (0..1).map(|thx| {
             let (scheduled_ee_receiver, completed_ee_sender) = (scheduled_ee_receiver.clone(), completed_ee_sender.clone());
             let bank = bank.clone();
 
@@ -4033,7 +4033,12 @@ impl Bank {
     /// reaches its max tick height. Can be called by tests to get new blockhashes for transaction
     /// processing without advancing to a new bank slot.
     pub fn register_recent_blockhash(&self, blockhash: &Hash) {
+        assert!(
+            !self.freeze_started(),
+            "register_tick() working on a bank that is already frozen or is undergoing freezing!"
+        );
         let mut scheduler = self.scheduler.write().unwrap();
+        assert!(scheduler.blockhash.is_none());
         scheduler.blockhash = Some(blockhash.clone());
     }
 

From 1378c296e07ed187b25f89ca65de0d0d9ca25931 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 6 Sep 2022 13:34:09 +0900
Subject: [PATCH 2192/3199] save

---
 runtime/src/bank.rs | 1 +
 1 file changed, 1 insertion(+)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index e86c4339772df0..6239c07a49781f 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -6403,6 +6403,7 @@ impl Bank {
             }
         };
 
+        assert!(scheduler.blockhash.is_none());
         for (st, &i) in batch.sanitized_transactions().iter().zip(transaction_indexes.iter()) {
             scheduler.schedule(st, i);
         }

From 2deb7fd3c5709de121701c1a84eabbda63cb6fe4 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 6 Sep 2022 13:37:56 +0900
Subject: [PATCH 2193/3199] save

---
 runtime/src/bank.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 6239c07a49781f..e5d995b9cbb09f 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1206,7 +1206,7 @@ struct Scheduler {
     graceful_stop_initiated: bool,
     errors: Arc<std::sync::Mutex<Vec<Result<()>>>>,
     bank: std::sync::Arc<std::sync::RwLock<std::option::Option<std::sync::Arc<Bank>>>>,
-    blockhash: Option<Hash>,
+    registered_blockhash: Option<Hash>,
 }
 
 impl Scheduler {

From 809c9b7d9233078f180cb07e2ee55b8636ac3d67 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 6 Sep 2022 13:38:21 +0900
Subject: [PATCH 2194/3199] save

---
 runtime/src/bank.rs | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index e5d995b9cbb09f..5205b88828b3b2 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1332,7 +1332,7 @@ impl Default for Scheduler {
             graceful_stop_initiated: Default::default(),
             errors,
             bank,
-            blockhash: Default::default(),
+            registered_blockhash: Default::default(),
         }
     }
 }
@@ -4038,8 +4038,8 @@ impl Bank {
             "register_tick() working on a bank that is already frozen or is undergoing freezing!"
         );
         let mut scheduler = self.scheduler.write().unwrap();
-        assert!(scheduler.blockhash.is_none());
-        scheduler.blockhash = Some(blockhash.clone());
+        assert!(scheduler.registered_blockhash.is_none());
+        scheduler.registered_blockhash = Some(blockhash.clone());
     }
 
     /// Tell the bank which Entry IDs exist on the ledger. This function assumes subsequent calls

From 77ae6ba9069ba83890d21b87ca069d46cde6bec2 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 6 Sep 2022 13:38:46 +0900
Subject: [PATCH 2195/3199] save

---
 runtime/src/bank.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 5205b88828b3b2..9dbd287681afb8 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -6403,7 +6403,7 @@ impl Bank {
             }
         };
 
-        assert!(scheduler.blockhash.is_none());
+        assert!(scheduler.registered_blockhash.is_none());
         for (st, &i) in batch.sanitized_transactions().iter().zip(transaction_indexes.iter()) {
             scheduler.schedule(st, i);
         }
@@ -8072,7 +8072,7 @@ impl Bank {
         // readers can starve this write lock acquisition and ticks would be slowed down too
         // much if the write lock is acquired for each tick.
         let mut w_blockhash_queue = self.blockhash_queue.write().unwrap();
-        w_blockhash_queue.register_hash(&scheduler.blockhash.unwrap(), self.fee_rate_governor.lamports_per_signature);
+        w_blockhash_queue.register_hash(&scheduler.registered_blockhash.unwrap(), self.fee_rate_governor.lamports_per_signature);
         self.update_recent_blockhashes_locked(&w_blockhash_queue);
         Ok(())
     }

From 40ff80da93da1f7c80152632ea269c9306c8a1bd Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 6 Sep 2022 13:47:33 +0900
Subject: [PATCH 2196/3199] save

---
 runtime/src/bank.rs | 28 ++++++++--------------------
 1 file changed, 8 insertions(+), 20 deletions(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 9dbd287681afb8..58a465fd3c7d2c 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1206,7 +1206,6 @@ struct Scheduler {
     graceful_stop_initiated: bool,
     errors: Arc<std::sync::Mutex<Vec<Result<()>>>>,
     bank: std::sync::Arc<std::sync::RwLock<std::option::Option<std::sync::Arc<Bank>>>>,
-    registered_blockhash: Option<Hash>,
 }
 
 impl Scheduler {
@@ -1244,7 +1243,7 @@ impl Default for Scheduler {
 
         let bank = Arc::new(std::sync::RwLock::new(None::<Arc<Bank>>));
 
-        let executing_thread_handles = (0..1).map(|thx| {
+        let executing_thread_handles = (0..8).map(|thx| {
             let (scheduled_ee_receiver, completed_ee_sender) = (scheduled_ee_receiver.clone(), completed_ee_sender.clone());
             let bank = bank.clone();
 
@@ -1332,7 +1331,6 @@ impl Default for Scheduler {
             graceful_stop_initiated: Default::default(),
             errors,
             bank,
-            registered_blockhash: Default::default(),
         }
     }
 }
@@ -4033,13 +4031,12 @@ impl Bank {
     /// reaches its max tick height. Can be called by tests to get new blockhashes for transaction
     /// processing without advancing to a new bank slot.
     pub fn register_recent_blockhash(&self, blockhash: &Hash) {
-        assert!(
-            !self.freeze_started(),
-            "register_tick() working on a bank that is already frozen or is undergoing freezing!"
-        );
-        let mut scheduler = self.scheduler.write().unwrap();
-        assert!(scheduler.registered_blockhash.is_none());
-        scheduler.registered_blockhash = Some(blockhash.clone());
+        // Only acquire the write lock for the blockhash queue on block boundaries because
+        // readers can starve this write lock acquisition and ticks would be slowed down too
+        // much if the write lock is acquired for each tick.
+        let mut w_blockhash_queue = self.blockhash_queue.write().unwrap();
+        w_blockhash_queue.register_hash(blockhash, self.fee_rate_governor.lamports_per_signature);
+        self.update_recent_blockhashes_locked(&w_blockhash_queue);
     }
 
     /// Tell the bank which Entry IDs exist on the ledger. This function assumes subsequent calls
@@ -6403,7 +6400,6 @@ impl Bank {
             }
         };
 
-        assert!(scheduler.registered_blockhash.is_none());
         for (st, &i) in batch.sanitized_transactions().iter().zip(transaction_indexes.iter()) {
             scheduler.schedule(st, i);
         }
@@ -8066,15 +8062,7 @@ impl Bank {
     pub fn wait_for_scheduler(&self) -> Result<()> {
         let mut scheduler = self.scheduler.write().unwrap();
         scheduler.gracefully_stop()?;
-        scheduler.handle_aborted_executions().into_iter().next().unwrap_or(Ok(()))?;
-
-        // Only acquire the write lock for the blockhash queue on block boundaries because
-        // readers can starve this write lock acquisition and ticks would be slowed down too
-        // much if the write lock is acquired for each tick.
-        let mut w_blockhash_queue = self.blockhash_queue.write().unwrap();
-        w_blockhash_queue.register_hash(&scheduler.registered_blockhash.unwrap(), self.fee_rate_governor.lamports_per_signature);
-        self.update_recent_blockhashes_locked(&w_blockhash_queue);
-        Ok(())
+        scheduler.handle_aborted_executions().into_iter().next().unwrap_or(Ok(()))
     }
 }
 

From 5209d4dfd63ef0a5502cc0eea197cfdece1e13ec Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 6 Sep 2022 13:47:44 +0900
Subject: [PATCH 2197/3199] save

---
 runtime/src/bank.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 58a465fd3c7d2c..38dd0d82d82b9d 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1243,7 +1243,7 @@ impl Default for Scheduler {
 
         let bank = Arc::new(std::sync::RwLock::new(None::<Arc<Bank>>));
 
-        let executing_thread_handles = (0..8).map(|thx| {
+        let executing_thread_handles = (0..1).map(|thx| {
             let (scheduled_ee_receiver, completed_ee_sender) = (scheduled_ee_receiver.clone(), completed_ee_sender.clone());
             let bank = bank.clone();
 

From 158596b2c284666af93ed360d0698511cb29913d Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 6 Sep 2022 13:56:14 +0900
Subject: [PATCH 2198/3199] save

---
 runtime/src/bank.rs | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 38dd0d82d82b9d..1a21d24a43d838 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -4031,10 +4031,18 @@ impl Bank {
     /// reaches its max tick height. Can be called by tests to get new blockhashes for transaction
     /// processing without advancing to a new bank slot.
     pub fn register_recent_blockhash(&self, blockhash: &Hash) {
+        info!("register_recent_blockhash: slot: {} reinitializaing the scheduler: start", self.slot());
+
+        self.wait_for_scheduler().unwrap();
+
         // Only acquire the write lock for the blockhash queue on block boundaries because
         // readers can starve this write lock acquisition and ticks would be slowed down too
         // much if the write lock is acquired for each tick.
         let mut w_blockhash_queue = self.blockhash_queue.write().unwrap();
+        self.scheduler.write().unwrap() = Default::default();
+
+        info!("register_recent_blockhash: slot: {} reinitializaing the scheduler: end", self.slot());
+
         w_blockhash_queue.register_hash(blockhash, self.fee_rate_governor.lamports_per_signature);
         self.update_recent_blockhashes_locked(&w_blockhash_queue);
     }

From 0ff075ba7f7182f55c046cb12d9fe8afd53b1d34 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 6 Sep 2022 13:56:58 +0900
Subject: [PATCH 2199/3199] save

---
 runtime/src/bank.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 1a21d24a43d838..65e44d35d6ac30 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -4039,7 +4039,7 @@ impl Bank {
         // readers can starve this write lock acquisition and ticks would be slowed down too
         // much if the write lock is acquired for each tick.
         let mut w_blockhash_queue = self.blockhash_queue.write().unwrap();
-        self.scheduler.write().unwrap() = Default::default();
+        *self.scheduler.write().unwrap() = Default::default();
 
         info!("register_recent_blockhash: slot: {} reinitializaing the scheduler: end", self.slot());
 

From 7adccaae03406416fb02d6274ac441319885ddcc Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 6 Sep 2022 14:35:52 +0900
Subject: [PATCH 2200/3199] save

---
 ledger/src/blockstore_processor.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ledger/src/blockstore_processor.rs b/ledger/src/blockstore_processor.rs
index 97ab0dc1ecc553..aa4a267a8d78ac 100644
--- a/ledger/src/blockstore_processor.rs
+++ b/ledger/src/blockstore_processor.rs
@@ -301,7 +301,7 @@ fn execute_batches_internal(
     let mut execute_batches_elapsed = Measure::start("execute_batches_elapsed");
     let results: Vec<Result<()>> = PAR_THREAD_POOL.install(|| {
         batches
-            .into_par_iter()
+            .into_iter()
             .enumerate()
             .map(|(index, transaction_batch_with_indexes)| {
                 let transaction_count = transaction_batch_with_indexes

From b7d25644b2e762ffaac7ac890633aec83b147cfe Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 6 Sep 2022 14:37:03 +0900
Subject: [PATCH 2201/3199] save

---
 ledger/src/blockstore_processor.rs | 26 ++------------------------
 1 file changed, 2 insertions(+), 24 deletions(-)

diff --git a/ledger/src/blockstore_processor.rs b/ledger/src/blockstore_processor.rs
index aa4a267a8d78ac..0f48e2c742d35d 100644
--- a/ledger/src/blockstore_processor.rs
+++ b/ledger/src/blockstore_processor.rs
@@ -299,7 +299,7 @@ fn execute_batches_internal(
         Mutex::new(HashMap::new());
 
     let mut execute_batches_elapsed = Measure::start("execute_batches_elapsed");
-    let results: Vec<Result<()>> = PAR_THREAD_POOL.install(|| {
+    let results: Vec<Result<()>> = 
         batches
             .into_iter()
             .enumerate()
@@ -329,32 +329,10 @@ fn execute_batches_internal(
                     "execute_batch",
                 );
 
-                let thread_index = PAR_THREAD_POOL.current_thread_index().unwrap();
-                execution_timings_per_thread
-                    .lock()
-                    .unwrap()
-                    .entry(thread_index)
-                    .and_modify(|thread_execution_time| {
-                        let ThreadExecuteTimings {
-                            total_thread_us,
-                            total_transactions_executed,
-                            execute_timings: total_thread_execute_timings,
-                        } = thread_execution_time;
-                        *total_thread_us += execute_batches_time.as_us();
-                        *total_transactions_executed += transaction_count;
-                        total_thread_execute_timings
-                            .saturating_add_in_place(ExecuteTimingType::TotalBatchesLen, 1);
-                        total_thread_execute_timings.accumulate(&timings);
-                    })
-                    .or_insert(ThreadExecuteTimings {
-                        total_thread_us: execute_batches_time.as_us(),
-                        total_transactions_executed: transaction_count,
-                        execute_timings: timings,
-                    });
                 result
             })
             .collect()
-    });
+    ;
     execute_batches_elapsed.stop();
 
     first_err(&results)?;

From 72ccb5a303d4ed385cf31815ff9d543422e09f6d Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 6 Sep 2022 15:12:08 +0900
Subject: [PATCH 2202/3199] save

---
 runtime/src/bank.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 65e44d35d6ac30..aef1cc53c8a252 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1307,7 +1307,7 @@ impl Default for Scheduler {
             let mut runnable_queue = solana_scheduler::TaskQueue::default();
 
             solana_scheduler::ScheduleStage::run(
-                123,
+                1,
                 &mut runnable_queue,
                 &mut address_book,
                 &transaction_receiver,

From 517e78ae2d4f27d18818ae7a12a273c05bf069f9 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 6 Sep 2022 16:50:50 +0900
Subject: [PATCH 2203/3199] save

---
 runtime/src/bank.rs  |  7 +++++++
 scheduler/src/lib.rs | 14 ++++++++------
 2 files changed, 15 insertions(+), 6 deletions(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index aef1cc53c8a252..4f399b7519ebfd 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1246,9 +1246,13 @@ impl Default for Scheduler {
         let executing_thread_handles = (0..1).map(|thx| {
             let (scheduled_ee_receiver, completed_ee_sender) = (scheduled_ee_receiver.clone(), completed_ee_sender.clone());
             let bank = bank.clone();
+            let mut execute_time = 0;
 
             std::thread::Builder::new().name(format!("solExec{:02}", thx)).spawn(move || {
             while let Ok(solana_scheduler::ExecutablePayload(mut ee)) = scheduled_ee_receiver.recv() {
+                assert_eq!(ee.task.execute_time(), execute_time);
+                execute_time += 1;
+
                 let bank_r = bank.read().unwrap();
                 let bank_o = (&bank_r).as_ref().unwrap();
                 let bank = bank_o.as_ref();
@@ -1272,10 +1276,13 @@ impl Default for Scheduler {
                     &mut Default::default(),
                     None
                 );
+                drop(bank);
+
                 let TransactionResults {
                     fee_collection_results,
                     ..
                 } = tx_results;
+
                 ee.execution_result = Some(fee_collection_results.into_iter().collect::<Result<_>>());
 
 
diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 482ee46d6f6ece..eef2009d398353 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -616,7 +616,7 @@ impl Task {
 
     pub fn record_execute_time(&self, queue_clock: usize, execute_clock: usize) {
         //self.queue_end_time.store(queue_clock, std::sync::atomic::Ordering::SeqCst);
-        //self.execute_time.store(execute_clock, std::sync::atomic::Ordering::SeqCst);
+        self.execute_time.store(execute_clock, std::sync::atomic::Ordering::SeqCst);
     }
 
     pub fn execute_time(&self) -> usize {
@@ -1242,7 +1242,7 @@ impl ScheduleStage {
         ast: AST,
         ee: &mut ExecutionEnvironment,
         address_book: &mut AddressBook,
-        commit_time: &mut usize,
+        commit_clock: &mut usize,
         provisioning_tracker_count: &mut usize,
     ) {
         // do par()-ly?
@@ -1250,9 +1250,10 @@ impl ScheduleStage {
         ee.reindex_with_address_book();
         assert!(ee.is_reindexed());
 
-        ee.task.record_commit_time(*commit_time);
+        ee.task.record_commit_time(*commit_clock);
         //ee.task.trace_timestamps("commit");
-        //*commit_time = commit_time.checked_add(1).unwrap();
+        assert_eq!(ee.task.execute_time(), commit_clock);
+        *commit_clock = commit_clock.checked_add(1).unwrap();
 
         // which order for data race free?: unlocking / marking
         Self::unlock_after_execution(
@@ -1335,6 +1336,7 @@ impl ScheduleStage {
         let mut sequence_time = 0;
         let mut queue_clock = 0;
         let mut execute_clock = 0;
+        let mut commit_clock = 0;
         let mut completed_count = 0_usize;
         assert!(max_executing_queue_count > 0);
 
@@ -1409,7 +1411,7 @@ impl ScheduleStage {
                    let mut processed_execution_environment = maybe_from_exec.unwrap().0;
                     executing_queue_count = executing_queue_count.checked_sub(1).unwrap();
                     completed_count = completed_count.checked_add(1).unwrap();
-                    Self::commit_completed_execution(ast, &mut processed_execution_environment, address_book, &mut execute_clock, &mut provisioning_tracker_count);
+                    Self::commit_completed_execution(ast, &mut processed_execution_environment, address_book, &mut commit_clock, &mut provisioning_tracker_count);
                     to_next_stage.send(DroppablePayload(processed_execution_environment)).unwrap();
                }
                recv(from_prev) -> maybe_from => {
@@ -1487,7 +1489,7 @@ impl ScheduleStage {
                             ast,
                             &mut processed_execution_environment,
                             address_book,
-                            &mut execute_clock,
+                            &mut commit_clock,
                             &mut provisioning_tracker_count,
                         );
                         to_next_stage.send(DroppablePayload(processed_execution_environment)).unwrap();

From 899a94a189163c8c0077ef35890de7b17c66b0a3 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 6 Sep 2022 16:51:27 +0900
Subject: [PATCH 2204/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index eef2009d398353..f57c0c75e4f07c 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1252,7 +1252,7 @@ impl ScheduleStage {
 
         ee.task.record_commit_time(*commit_clock);
         //ee.task.trace_timestamps("commit");
-        assert_eq!(ee.task.execute_time(), commit_clock);
+        assert_eq!(ee.task.execute_time(), *commit_clock);
         *commit_clock = commit_clock.checked_add(1).unwrap();
 
         // which order for data race free?: unlocking / marking

From d9aa3a4db6d13c1afd1902feef0207e4deb6fdba Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 6 Sep 2022 17:18:22 +0900
Subject: [PATCH 2205/3199] save

---
 runtime/src/bank.rs  | 8 +++++++-
 scheduler/src/lib.rs | 4 ++--
 2 files changed, 9 insertions(+), 3 deletions(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 4f399b7519ebfd..7197da7f91df7c 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1206,6 +1206,7 @@ struct Scheduler {
     graceful_stop_initiated: bool,
     errors: Arc<std::sync::Mutex<Vec<Result<()>>>>,
     bank: std::sync::Arc<std::sync::RwLock<std::option::Option<std::sync::Arc<Bank>>>>,
+    transaction_index: AtomicUsize,
 }
 
 impl Scheduler {
@@ -1226,6 +1227,7 @@ impl Scheduler {
             .map(|address| solana_scheduler::LockAttempt::new(self.preloader.load(**address), solana_scheduler::RequestedUsage::Readonly));
         let locks = writable_lock_iter.chain(readonly_lock_iter).collect::<Vec<_>>();
 
+        assert_eq!(index, self.transaction_index.fetch_add(1, std::sync::atomic::Ordering::SeqCst));
         let uw = usize::max_value() - index;
         let t = solana_scheduler::Task::new_for_queue(nast, uw as u64, (sanitized_tx.clone(), locks));
         self.transaction_sender.as_ref().unwrap().send(solana_scheduler::SchedulablePayload(t)).unwrap();
@@ -1312,9 +1314,13 @@ impl Default for Scheduler {
 
         let scheduler_thread_handle = std::thread::Builder::new().name("solScheduler".to_string()).spawn(move || {
             let mut runnable_queue = solana_scheduler::TaskQueue::default();
+            let max_executing_queue_count = std::env::var("MAX_EXECUTING_QUEUE_COUNT")
+                .unwrap_or(format!("{}", 1))
+                .parse::<usize>()
+                .unwrap();
 
             solana_scheduler::ScheduleStage::run(
-                1,
+                max_executing_queue_count,
                 &mut runnable_queue,
                 &mut address_book,
                 &transaction_receiver,
diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index f57c0c75e4f07c..ecd85383ce08e2 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1065,7 +1065,7 @@ impl ScheduleStage {
                     );
 
                     break;
-                    continue;
+                    //continue;
                 }
 
                 trace!(
@@ -1443,7 +1443,7 @@ impl ScheduleStage {
                         start = std::time::Instant::now();
                         info!("schedule_once:interval (from: {}, to: {}, runnnable: {}, contended: {}, (immediate+provisional)/max: ({}+{})/{}) active from contended: {} stuck: {} completed: {}!", from_prev.len(), to_execute_substage.len(), runnable_queue.task_count(), contended_count, executing_queue_count, provisioning_tracker_count, max_executing_queue_count, address_book.uncontended_task_ids.len(), address_book.stuck_tasks.len(), completed_count);
                     }
-                    let prefer_immediate = provisioning_tracker_count / 4 > executing_queue_count;
+                    let prefer_immediate = true; // provisioning_tracker_count / 4 > executing_queue_count;
                     if let Some(ee) = Self::schedule_next_execution(
                         ast,
                         &task_sender,

From dac5184d802a2f4eaece2583db085860395341cf Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 6 Sep 2022 17:19:08 +0900
Subject: [PATCH 2206/3199] save

---
 runtime/src/bank.rs | 1 +
 1 file changed, 1 insertion(+)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 7197da7f91df7c..15157cbc7633fc 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1344,6 +1344,7 @@ impl Default for Scheduler {
             graceful_stop_initiated: Default::default(),
             errors,
             bank,
+            transaction_index: Default::default(),
         }
     }
 }

From 6c9c28ed8cba9479c64f6ae750edb0caa24b8b46 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 6 Sep 2022 17:20:21 +0900
Subject: [PATCH 2207/3199] save

---
 runtime/src/bank.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 15157cbc7633fc..4fc52f5b72cc3e 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1227,7 +1227,7 @@ impl Scheduler {
             .map(|address| solana_scheduler::LockAttempt::new(self.preloader.load(**address), solana_scheduler::RequestedUsage::Readonly));
         let locks = writable_lock_iter.chain(readonly_lock_iter).collect::<Vec<_>>();
 
-        assert_eq!(index, self.transaction_index.fetch_add(1, std::sync::atomic::Ordering::SeqCst));
+        //assert_eq!(index, self.transaction_index.fetch_add(1, std::sync::atomic::Ordering::SeqCst));
         let uw = usize::max_value() - index;
         let t = solana_scheduler::Task::new_for_queue(nast, uw as u64, (sanitized_tx.clone(), locks));
         self.transaction_sender.as_ref().unwrap().send(solana_scheduler::SchedulablePayload(t)).unwrap();

From ce8dbe8f722ab5fa387f7ae63b6a48cf0bd206bf Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 6 Sep 2022 17:33:57 +0900
Subject: [PATCH 2208/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index ecd85383ce08e2..2dc6e0edcc6560 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -962,7 +962,7 @@ impl ScheduleStage {
                         &mut next_task.lock_attempts_mut(ast),
                     );
 
-                if unlockable_count > 0 {
+                if unlockable_count > 0 || from_runnable {
                     //trace!("reset_lock_for_failed_execution(): {:?} {}", (&unique_weight, from_runnable), next_task.tx.0.signature());
                     Self::reset_lock_for_failed_execution(
                         ast,

From 59c10e75354e6d244425df9350c6b858a49e844a Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 6 Sep 2022 21:29:17 +0900
Subject: [PATCH 2209/3199] save

---
 runtime/src/bank.rs | 11 ++++++++---
 1 file changed, 8 insertions(+), 3 deletions(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 4fc52f5b72cc3e..41ab408e58af50 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1245,7 +1245,12 @@ impl Default for Scheduler {
 
         let bank = Arc::new(std::sync::RwLock::new(None::<Arc<Bank>>));
 
-        let executing_thread_handles = (0..1).map(|thx| {
+        let executing_thread_count = std::env::var("EXECUTING_THREAD_COUNT")
+            .unwrap_or(format!("{}", 1))
+            .parse::<usize>()
+            .unwrap();
+
+        let executing_thread_handles = (0..executing_thread_count).map(|thx| {
             let (scheduled_ee_receiver, completed_ee_sender) = (scheduled_ee_receiver.clone(), completed_ee_sender.clone());
             let bank = bank.clone();
             let mut execute_time = 0;
@@ -4045,7 +4050,7 @@ impl Bank {
     /// reaches its max tick height. Can be called by tests to get new blockhashes for transaction
     /// processing without advancing to a new bank slot.
     pub fn register_recent_blockhash(&self, blockhash: &Hash) {
-        info!("register_recent_blockhash: slot: {} reinitializaing the scheduler: start", self.slot());
+        info!("register_recent_blockhash: slot: {} reinitializing the scheduler: start", self.slot());
 
         self.wait_for_scheduler().unwrap();
 
@@ -4055,7 +4060,7 @@ impl Bank {
         let mut w_blockhash_queue = self.blockhash_queue.write().unwrap();
         *self.scheduler.write().unwrap() = Default::default();
 
-        info!("register_recent_blockhash: slot: {} reinitializaing the scheduler: end", self.slot());
+        info!("register_recent_blockhash: slot: {} reinitializing the scheduler: end", self.slot());
 
         w_blockhash_queue.register_hash(blockhash, self.fee_rate_governor.lamports_per_signature);
         self.update_recent_blockhashes_locked(&w_blockhash_queue);

From af6a5fa397866531fe0da4ed403db11474a7e1c5 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 6 Sep 2022 21:35:54 +0900
Subject: [PATCH 2210/3199] save

---
 runtime/src/bank.rs | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 41ab408e58af50..8fe9d4d876beea 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1257,8 +1257,10 @@ impl Default for Scheduler {
 
             std::thread::Builder::new().name(format!("solExec{:02}", thx)).spawn(move || {
             while let Ok(solana_scheduler::ExecutablePayload(mut ee)) = scheduled_ee_receiver.recv() {
-                assert_eq!(ee.task.execute_time(), execute_time);
+                let current_execute_clock = ee.task.execute_time();
+                assert_eq!(current_execute_clock, execute_time);
                 execute_time += 1;
+                trace!("executing thread: {} transaction_index: {} execute_clock: {}", thx, ee.task.transaction_index_in_entries_for_replay(), current_execute_clock);
 
                 let bank_r = bank.read().unwrap();
                 let bank_o = (&bank_r).as_ref().unwrap();

From 535935b377f6f9f6e03d085a5627ef79876110ba Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 6 Sep 2022 21:36:02 +0900
Subject: [PATCH 2211/3199] save

---
 scheduler/src/lib.rs | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 2dc6e0edcc6560..94aad7c8e4ac5f 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -563,6 +563,10 @@ impl Task {
         })
     }
 
+    pub fn transaction_index_in_entries_for_replay() -> u64 {
+        u64::max_value() - self.unique_weight
+    }
+
     #[inline(never)]
     pub fn clone_in_queue(this: &TaskInQueue) -> TaskInQueue {
         TaskInQueue::clone(this)

From 5ad5f41332ec5ab7dffd269922cc82bc77617a0e Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 6 Sep 2022 21:36:11 +0900
Subject: [PATCH 2212/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 94aad7c8e4ac5f..cd09a5c2204793 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -563,7 +563,7 @@ impl Task {
         })
     }
 
-    pub fn transaction_index_in_entries_for_replay() -> u64 {
+    pub fn transaction_index_in_entries_for_replay(&self) -> u64 {
         u64::max_value() - self.unique_weight
     }
 

From d124dde7653c087437f90b94b37426820dacaf6a Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 6 Sep 2022 21:41:49 +0900
Subject: [PATCH 2213/3199] save

---
 runtime/src/bank.rs | 27 ++++++++++++++++++++++++++-
 1 file changed, 26 insertions(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 8fe9d4d876beea..a22320fdf3d8e3 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1250,15 +1250,23 @@ impl Default for Scheduler {
             .parse::<usize>()
             .unwrap();
 
+        let send_metrics = std::env::var("SEND_METRICS").is_ok();
+        let disable_clock_asserts = std::env::var("DISABLE_CLOCK_ASSERTS").is_ok();
+
         let executing_thread_handles = (0..executing_thread_count).map(|thx| {
             let (scheduled_ee_receiver, completed_ee_sender) = (scheduled_ee_receiver.clone(), completed_ee_sender.clone());
             let bank = bank.clone();
             let mut execute_time = 0;
 
             std::thread::Builder::new().name(format!("solExec{:02}", thx)).spawn(move || {
+            let current_thread_name = std::thread::current().name().unwrap().to_string();
             while let Ok(solana_scheduler::ExecutablePayload(mut ee)) = scheduled_ee_receiver.recv() {
+                let mut process_message_time = Measure::start("process_message_time");
+
                 let current_execute_clock = ee.task.execute_time();
-                assert_eq!(current_execute_clock, execute_time);
+                if !disable_clock_asserts {
+                    assert_eq!(current_execute_clock, execute_time);
+                }
                 execute_time += 1;
                 trace!("executing thread: {} transaction_index: {} execute_clock: {}", thx, ee.task.transaction_index_in_entries_for_replay(), current_execute_clock);
 
@@ -1296,6 +1304,23 @@ impl Default for Scheduler {
 
 
                 //ee.reindex_with_address_book();
+                if send_metrics {
+                    let sig = ee.task.tx.0.signature().to_string();
+
+                    process_message_time.stop();
+                    let duration_with_overhead = process_message_time.as_us();
+
+                    datapoint_info!(
+                        "individual_tx_stats",
+                        ("slot", bank.slot(), i64),
+                        ("thread", current_thread_name, String),
+                        ("signature", &sig, String),
+                        ("account_locks_in_json", "{}", String),
+                        ("status", "Ok", String),
+                        ("duration", duration_with_overhead, i64),
+                        ("compute_units", ee.cu, i64),
+                    );
+                }
                 completed_ee_sender.send(solana_scheduler::UnlockablePayload(ee)).unwrap();
             }
             Ok(())

From ada2a7d588c0495449505e21b0186730c2136519 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 6 Sep 2022 21:49:31 +0900
Subject: [PATCH 2214/3199] save

---
 ledger/src/blockstore_processor.rs | 26 +++++++++++++++++++++++++-
 runtime/src/bank.rs                |  4 ++--
 2 files changed, 27 insertions(+), 3 deletions(-)

diff --git a/ledger/src/blockstore_processor.rs b/ledger/src/blockstore_processor.rs
index 0f48e2c742d35d..d2799b8019d9cf 100644
--- a/ledger/src/blockstore_processor.rs
+++ b/ledger/src/blockstore_processor.rs
@@ -178,15 +178,17 @@ fn execute_batch(
         batch,
         transaction_indexes,
     } = batch;
-    let record_token_balances = transaction_status_sender.is_some();
+    //let record_token_balances = transaction_status_sender.is_some();
 
     let mut mint_decimals: HashMap<Pubkey, u8> = HashMap::new();
 
+    /*
     let pre_token_balances = if record_token_balances {
         collect_token_balances(bank, batch, &mut mint_decimals)
     } else {
         vec![]
     };
+    */
 
     let pre_process_units: u64 = aggregate_total_execution_units(timings);
 
@@ -329,6 +331,28 @@ fn execute_batches_internal(
                     "execute_batch",
                 );
 
+                let thread_index = 0;// PAR_THREAD_POOL.current_thread_index().unwrap();
+                execution_timings_per_thread
+                    .lock()
+                    .unwrap()
+                    .entry(thread_index)
+                    .and_modify(|thread_execution_time| {
+                        let ThreadExecuteTimings {
+                            total_thread_us,
+                            total_transactions_executed,
+                            execute_timings: total_thread_execute_timings,
+                        } = thread_execution_time;
+                        *total_thread_us += execute_batches_time.as_us();
+                        *total_transactions_executed += transaction_count;
+                        total_thread_execute_timings
+                            .saturating_add_in_place(ExecuteTimingType::TotalBatchesLen, 1);
+                        total_thread_execute_timings.accumulate(&timings);
+                    })
+                    .or_insert(ThreadExecuteTimings {
+                        total_thread_us: execute_batches_time.as_us(),
+                        total_transactions_executed: transaction_count,
+                        execute_timings: timings,
+                    });
                 result
             })
             .collect()
diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index a22320fdf3d8e3..a8e651d947db2f 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1260,6 +1260,7 @@ impl Default for Scheduler {
 
             std::thread::Builder::new().name(format!("solExec{:02}", thx)).spawn(move || {
             let current_thread_name = std::thread::current().name().unwrap().to_string();
+
             while let Ok(solana_scheduler::ExecutablePayload(mut ee)) = scheduled_ee_receiver.recv() {
                 let mut process_message_time = Measure::start("process_message_time");
 
@@ -1268,7 +1269,7 @@ impl Default for Scheduler {
                     assert_eq!(current_execute_clock, execute_time);
                 }
                 execute_time += 1;
-                trace!("executing thread: {} transaction_index: {} execute_clock: {}", thx, ee.task.transaction_index_in_entries_for_replay(), current_execute_clock);
+                trace!("execute_substage: thread: {} transaction_index: {} execute_clock: {}", thx, ee.task.transaction_index_in_entries_for_replay(), current_execute_clock);
 
                 let bank_r = bank.read().unwrap();
                 let bank_o = (&bank_r).as_ref().unwrap();
@@ -1282,7 +1283,6 @@ impl Default for Scheduler {
                     TransactionBatch::new(vec![lock_result], &bank, Cow::Owned(vec![ee.task.tx.0.clone()]));
                 batch.set_needs_unlock(false);
 
-
                 let (tx_results, _balances) = bank.load_execute_and_commit_transactions(
                     &batch,
                     MAX_PROCESSING_AGE,

From a034fa26551cf82fe78450729b2b6b645ad58e24 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 6 Sep 2022 22:00:54 +0900
Subject: [PATCH 2215/3199] save

---
 scheduler/src/lib.rs | 19 +++++++++++++------
 1 file changed, 13 insertions(+), 6 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index cd09a5c2204793..3217e268bb50a6 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -64,7 +64,7 @@ impl ExecutionEnvironment {
             .load(std::sync::atomic::Ordering::SeqCst)
             > 0;
         for mut lock_attempt in self.finalized_lock_attempts.iter_mut() {
-            let contended_unique_weights = lock_attempt.contended_unique_weights();
+            let contended_unique_weights = lock_attempt.target_contended_unique_weights();
             contended_unique_weights
                 .heaviest_task_cursor()
                 .map(|mut task_cursor| {
@@ -165,8 +165,8 @@ impl LockAttempt {
         }
     }
 
-    pub fn contended_unique_weights(&self) -> &TaskIds {
-        &self.target.0 .1
+    pub fn target_contended_unique_weights(&self) -> &TaskIds {
+        &self.target.0.1
     }
 }
 
@@ -327,6 +327,13 @@ impl AddressBook {
 
         let mut page = target.page_mut(ast);
 
+        let strictly_lockable = page.contended_unique_weights.is_empty() ||
+            page.contended_unique_weights.task_ids.back().key() == unique_weight
+        if !strictly_lockable {
+            *status = LockStatus::Failed;
+            return;
+        }
+
         let next_usage = page.next_usage;
         match page.current_usage {
             Usage::Unused => {
@@ -717,7 +724,7 @@ impl Task {
         task_sender: &crossbeam_channel::Sender<(TaskInQueue, Vec<LockAttempt>)>,
     ) {
         for lock_attempt in this.lock_attempts_mut(ast).iter() {
-            lock_attempt.contended_unique_weights().insert_task(this.unique_weight, Task::clone_in_queue(this));
+            lock_attempt.target_contended_unique_weights().insert_task(this.unique_weight, Task::clone_in_queue(this));
         }
         //let a = Task::clone_in_queue(this);
         //task_sender
@@ -966,7 +973,7 @@ impl ScheduleStage {
                         &mut next_task.lock_attempts_mut(ast),
                     );
 
-                if unlockable_count > 0 || from_runnable {
+                if unlockable_count > 0 {
                     //trace!("reset_lock_for_failed_execution(): {:?} {}", (&unique_weight, from_runnable), next_task.tx.0.signature());
                     Self::reset_lock_for_failed_execution(
                         ast,
@@ -1388,7 +1395,7 @@ impl ScheduleStage {
                                 break;
                             }
                             lock_attempt
-                                .contended_unique_weights()
+                                .target_contended_unique_weights()
                                 .insert_task(task.unique_weight, Task::clone_in_queue(&task));
                         }
                     }

From 4bf5b9b8eb56939836adb7ba9a2f2b6db5ebfdb3 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 6 Sep 2022 22:01:09 +0900
Subject: [PATCH 2216/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 3217e268bb50a6..8b0e941402a4a5 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -328,7 +328,7 @@ impl AddressBook {
         let mut page = target.page_mut(ast);
 
         let strictly_lockable = page.contended_unique_weights.is_empty() ||
-            page.contended_unique_weights.task_ids.back().key() == unique_weight
+            page.contended_unique_weights.task_ids.back().key() == unique_weight;
         if !strictly_lockable {
             *status = LockStatus::Failed;
             return;

From 7cc3febe47a774539f13774eed31509a374b23a0 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 6 Sep 2022 22:02:30 +0900
Subject: [PATCH 2217/3199] save

---
 scheduler/src/lib.rs | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 8b0e941402a4a5..65e1cffc1b3bc5 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -123,7 +123,7 @@ pub unsafe trait NotAtScheduleThread: Copy {}
 
 impl PageRc {
     fn page_mut<AST: AtScheduleThread>(&self, _ast: AST) -> std::cell::RefMut<'_, Page> {
-        self.0 .0.borrow_mut()
+        self.0.0.borrow_mut()
     }
 }
 
@@ -325,15 +325,15 @@ impl AddressBook {
             ..
         } = attempt;
 
-        let mut page = target.page_mut(ast);
-
-        let strictly_lockable = page.contended_unique_weights.is_empty() ||
+        let strictly_lockable = target.0.1.contended_unique_weights.is_empty() ||
             page.contended_unique_weights.task_ids.back().key() == unique_weight;
         if !strictly_lockable {
             *status = LockStatus::Failed;
             return;
         }
 
+        let mut page = target.page_mut(ast);
+
         let next_usage = page.next_usage;
         match page.current_usage {
             Usage::Unused => {

From 261e2dd23ffcf04dcb27e38edd9220e39ebb798e Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 6 Sep 2022 22:02:49 +0900
Subject: [PATCH 2218/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 65e1cffc1b3bc5..370c70b53e56d8 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -325,7 +325,7 @@ impl AddressBook {
             ..
         } = attempt;
 
-        let strictly_lockable = target.0.1.contended_unique_weights.is_empty() ||
+        let strictly_lockable = target.0.1.is_empty() ||
             page.contended_unique_weights.task_ids.back().key() == unique_weight;
         if !strictly_lockable {
             *status = LockStatus::Failed;

From 9b7ceda518283df5dcac09ccc7a322c0d9595f0c Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 6 Sep 2022 22:03:41 +0900
Subject: [PATCH 2219/3199] save

---
 scheduler/src/lib.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 370c70b53e56d8..24f9b183755bef 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -325,8 +325,8 @@ impl AddressBook {
             ..
         } = attempt;
 
-        let strictly_lockable = target.0.1.is_empty() ||
-            page.contended_unique_weights.task_ids.back().key() == unique_weight;
+        let strictly_lockable = .is_empty() ||
+            target.0.1.back().key() == unique_weight;
         if !strictly_lockable {
             *status = LockStatus::Failed;
             return;

From 7db35a6610d8203b89e2a5ae9682d434ef6c8e56 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 6 Sep 2022 22:04:03 +0900
Subject: [PATCH 2220/3199] save

---
 scheduler/src/lib.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 24f9b183755bef..be7a3169038bbb 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -325,8 +325,8 @@ impl AddressBook {
             ..
         } = attempt;
 
-        let strictly_lockable = .is_empty() ||
-            target.0.1.back().key() == unique_weight;
+        let strictly_lockable = target.0.1.is_empty() ||
+            target.0.1.back().unwrap().key() == unique_weight;
         if !strictly_lockable {
             *status = LockStatus::Failed;
             return;

From 398ce814bbdff077243a91e778f5505e02ebb1a9 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 6 Sep 2022 22:04:15 +0900
Subject: [PATCH 2221/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index be7a3169038bbb..459820ffb91697 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -325,7 +325,7 @@ impl AddressBook {
             ..
         } = attempt;
 
-        let strictly_lockable = target.0.1.is_empty() ||
+        let strictly_lockable = target.0.1.task_ids.is_empty() ||
             target.0.1.back().unwrap().key() == unique_weight;
         if !strictly_lockable {
             *status = LockStatus::Failed;

From 394fe0298eef4974c8ab26db23673f65d9f29fb2 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 6 Sep 2022 22:04:26 +0900
Subject: [PATCH 2222/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 459820ffb91697..5f29f8a7a833ad 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -326,7 +326,7 @@ impl AddressBook {
         } = attempt;
 
         let strictly_lockable = target.0.1.task_ids.is_empty() ||
-            target.0.1.back().unwrap().key() == unique_weight;
+            target.0.1.task_ids.back().unwrap().key() == unique_weight;
         if !strictly_lockable {
             *status = LockStatus::Failed;
             return;

From 81ef0ccb4429c1369c857a51791a6b6508c95787 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 6 Sep 2022 22:04:50 +0900
Subject: [PATCH 2223/3199] save

---
 scheduler/src/lib.rs | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 5f29f8a7a833ad..76d1d42e8ca2ba 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -325,15 +325,15 @@ impl AddressBook {
             ..
         } = attempt;
 
+        let mut page = target.page_mut(ast);
+
         let strictly_lockable = target.0.1.task_ids.is_empty() ||
             target.0.1.task_ids.back().unwrap().key() == unique_weight;
         if !strictly_lockable {
             *status = LockStatus::Failed;
-            return;
+            return page.cu;
         }
 
-        let mut page = target.page_mut(ast);
-
         let next_usage = page.next_usage;
         match page.current_usage {
             Usage::Unused => {

From 10b579d30e57e53788fa8dff57e13516aaae9e5f Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 6 Sep 2022 22:06:30 +0900
Subject: [PATCH 2224/3199] save

---
 ledger-tool/src/main.rs | 1 +
 1 file changed, 1 insertion(+)

diff --git a/ledger-tool/src/main.rs b/ledger-tool/src/main.rs
index 3c2ae0568aec40..464feeb1ff3da7 100644
--- a/ledger-tool/src/main.rs
+++ b/ledger-tool/src/main.rs
@@ -378,6 +378,7 @@ fn output_slot(
                                 ()
                             });
                         }
+                        ee.task.reindex_with_address_book();
                         post_execute_env_sender.send(solana_scheduler::UnlockablePayload(ee)).unwrap();
                     }
                 })

From ee2e04316c149b354322d8984393c4462bbd0a48 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 6 Sep 2022 22:06:46 +0900
Subject: [PATCH 2225/3199] save

---
 ledger-tool/src/main.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ledger-tool/src/main.rs b/ledger-tool/src/main.rs
index 464feeb1ff3da7..c60db7d8e6d452 100644
--- a/ledger-tool/src/main.rs
+++ b/ledger-tool/src/main.rs
@@ -378,7 +378,7 @@ fn output_slot(
                                 ()
                             });
                         }
-                        ee.task.reindex_with_address_book();
+                        ee.reindex_with_address_book();
                         post_execute_env_sender.send(solana_scheduler::UnlockablePayload(ee)).unwrap();
                     }
                 })

From 9f6dab56ab15eb7d6ec897c79874340cbf07148a Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 6 Sep 2022 22:07:06 +0900
Subject: [PATCH 2226/3199] save

---
 ledger-tool/src/main.rs | 38 --------------------------------------
 1 file changed, 38 deletions(-)

diff --git a/ledger-tool/src/main.rs b/ledger-tool/src/main.rs
index c60db7d8e6d452..472a61ccafb9e0 100644
--- a/ledger-tool/src/main.rs
+++ b/ledger-tool/src/main.rs
@@ -340,44 +340,6 @@ fn output_slot(
                             .send(solana_scheduler::Multiplexed::FromExecute(ee))
                             .unwrap();
                         */
-                        let uq = ee.unique_weight;
-                        let should_remove = ee.task.contention_count() > 0;
-                        for mut lock_attempt in ee.finalized_lock_attempts.iter_mut() {
-                            let contended_unique_weights = lock_attempt.contended_unique_weights();
-                            contended_unique_weights.heaviest_task_cursor().map(|mut task_cursor| {
-                                let mut found = true;
-                                let mut removed = false;
-                                let mut task = task_cursor.value();
-                                //task.trace_timestamps("in_exec(initial list)");
-                                while !task.currently_contended() {
-                                    if task_cursor.key() == &uq {
-                                        assert!(should_remove);
-                                        removed = task_cursor.remove();
-                                        assert!(removed);
-                                    }
-                                    if task.already_finished() {
-                                        task_cursor.remove();
-                                    }
-                                    if let Some(new_cursor) = task_cursor.prev() {
-                                        assert!(new_cursor.key() < task_cursor.key());
-                                        task_cursor = new_cursor;
-                                        task = task_cursor.value();
-                                        //task.trace_timestamps("in_exec(subsequent list)");
-                                    } else {
-                                        found = false;
-                                        break;
-                                    }
-                                }
-                                if should_remove && !removed {
-                                    contended_unique_weights.remove_task(&uq);
-                                }
-                                found.then(|| solana_scheduler::TaskInQueue::clone(task))
-                            }).flatten().map(|task| {
-                                //task.trace_timestamps(&format!("in_exec(heaviest:{})", transaction_batch.task.queue_time_label()));
-                                lock_attempt.heaviest_uncontended = Some(task);
-                                ()
-                            });
-                        }
                         ee.reindex_with_address_book();
                         post_execute_env_sender.send(solana_scheduler::UnlockablePayload(ee)).unwrap();
                     }

From 8575f7f4b3550a340001feeca60d6ddda26a4c58 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 6 Sep 2022 22:14:01 +0900
Subject: [PATCH 2227/3199] save

---
 scheduler/src/lib.rs | 18 ++++++++++--------
 1 file changed, 10 insertions(+), 8 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 76d1d42e8ca2ba..02b56296dda7db 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -318,6 +318,16 @@ impl AddressBook {
         unique_weight: &UniqueWeight,
         attempt: &mut LockAttempt,
     ) -> CU {
+        let strictly_lockable_for_replay = attempt.target_contended_unique_weights().task_ids.is_empty() ||
+            attempt.target_contended_unique_weights().task_ids.back().unwrap().key() == unique_weight;
+        if !strictly_lockable_for_replay {
+            *status = LockStatus::Failed;
+            let page = attempt.target.page_mut(ast);
+            return page.cu;
+        }
+
+        let mut page = attempt.target.page_mut(ast);
+
         let LockAttempt {
             target,
             requested_usage,
@@ -325,14 +335,6 @@ impl AddressBook {
             ..
         } = attempt;
 
-        let mut page = target.page_mut(ast);
-
-        let strictly_lockable = target.0.1.task_ids.is_empty() ||
-            target.0.1.task_ids.back().unwrap().key() == unique_weight;
-        if !strictly_lockable {
-            *status = LockStatus::Failed;
-            return page.cu;
-        }
 
         let next_usage = page.next_usage;
         match page.current_usage {

From 6e301f796e446a9f44a62a4bd8a41a50b0a737ac Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 6 Sep 2022 22:14:23 +0900
Subject: [PATCH 2228/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 02b56296dda7db..975e5afe249e27 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -321,7 +321,7 @@ impl AddressBook {
         let strictly_lockable_for_replay = attempt.target_contended_unique_weights().task_ids.is_empty() ||
             attempt.target_contended_unique_weights().task_ids.back().unwrap().key() == unique_weight;
         if !strictly_lockable_for_replay {
-            *status = LockStatus::Failed;
+            attempt.status = LockStatus::Failed;
             let page = attempt.target.page_mut(ast);
             return page.cu;
         }

From 3243bcecc11f7c4e886405ce90af68b8be6188d3 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 6 Sep 2022 22:15:19 +0900
Subject: [PATCH 2229/3199] save

---
 scheduler/src/lib.rs | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 975e5afe249e27..7c7aa2e6c4bd2d 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -324,7 +324,7 @@ impl AddressBook {
             attempt.status = LockStatus::Failed;
             let page = attempt.target.page_mut(ast);
             return page.cu;
-        }
+        } else {
 
         let mut page = attempt.target.page_mut(ast);
 
@@ -391,6 +391,7 @@ impl AddressBook {
             }
         }
         page.cu
+        }
     }
 
     fn reset_lock<AST: AtScheduleThread>(

From 15d0df74b56dbb7fb2811353888ccb0545362c99 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 6 Sep 2022 22:16:06 +0900
Subject: [PATCH 2230/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 7c7aa2e6c4bd2d..d5bce1252fe542 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -326,7 +326,6 @@ impl AddressBook {
             return page.cu;
         } else {
 
-        let mut page = attempt.target.page_mut(ast);
 
         let LockAttempt {
             target,
@@ -334,6 +333,7 @@ impl AddressBook {
             status, /*, remembered*/
             ..
         } = attempt;
+        let mut page = target.page_mut(ast);
 
 
         let next_usage = page.next_usage;

From fa7eec2d446fb49d1e0705e7d4053d4087513450 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 6 Sep 2022 22:16:23 +0900
Subject: [PATCH 2231/3199] save

---
 scheduler/src/lib.rs | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index d5bce1252fe542..f9d29463baec66 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -324,7 +324,7 @@ impl AddressBook {
             attempt.status = LockStatus::Failed;
             let page = attempt.target.page_mut(ast);
             return page.cu;
-        } else {
+        }
 
 
         let LockAttempt {
@@ -391,7 +391,6 @@ impl AddressBook {
             }
         }
         page.cu
-        }
     }
 
     fn reset_lock<AST: AtScheduleThread>(

From 4a220d70809670efcd23a487f81d081f359a07a4 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 6 Sep 2022 22:37:20 +0900
Subject: [PATCH 2232/3199] save

---
 scheduler/src/lib.rs | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index f9d29463baec66..19bddc5cd6e781 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -320,6 +320,7 @@ impl AddressBook {
     ) -> CU {
         let strictly_lockable_for_replay = attempt.target_contended_unique_weights().task_ids.is_empty() ||
             attempt.target_contended_unique_weights().task_ids.back().unwrap().key() == unique_weight;
+
         if !strictly_lockable_for_replay {
             attempt.status = LockStatus::Failed;
             let page = attempt.target.page_mut(ast);
@@ -1456,7 +1457,7 @@ impl ScheduleStage {
                         start = std::time::Instant::now();
                         info!("schedule_once:interval (from: {}, to: {}, runnnable: {}, contended: {}, (immediate+provisional)/max: ({}+{})/{}) active from contended: {} stuck: {} completed: {}!", from_prev.len(), to_execute_substage.len(), runnable_queue.task_count(), contended_count, executing_queue_count, provisioning_tracker_count, max_executing_queue_count, address_book.uncontended_task_ids.len(), address_book.stuck_tasks.len(), completed_count);
                     }
-                    let prefer_immediate = true; // provisioning_tracker_count / 4 > executing_queue_count;
+                    let prefer_immediate = provisioning_tracker_count / 4 > executing_queue_count;
                     if let Some(ee) = Self::schedule_next_execution(
                         ast,
                         &task_sender,

From 49960f2e996f4965d3205d708d92d447b56bbfae Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 6 Sep 2022 22:44:37 +0900
Subject: [PATCH 2233/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 19bddc5cd6e781..28c0176458099e 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1266,7 +1266,7 @@ impl ScheduleStage {
 
         ee.task.record_commit_time(*commit_clock);
         //ee.task.trace_timestamps("commit");
-        assert_eq!(ee.task.execute_time(), *commit_clock);
+        //assert_eq!(ee.task.execute_time(), *commit_clock);
         *commit_clock = commit_clock.checked_add(1).unwrap();
 
         // which order for data race free?: unlocking / marking

From 4355496c85dc0e32044dabec003723a6553e291f Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Wed, 7 Sep 2022 09:23:39 +0900
Subject: [PATCH 2234/3199] save

---
 runtime/src/bank.rs  | 4 ++--
 scheduler/src/lib.rs | 4 ++--
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index a8e651d947db2f..c1474ecf71244d 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1258,7 +1258,7 @@ impl Default for Scheduler {
             let bank = bank.clone();
             let mut execute_time = 0;
 
-            std::thread::Builder::new().name(format!("solExec{:02}", thx)).spawn(move || {
+            std::thread::Builder::new().name(format!("solScExLane{:02}", thx)).spawn(move || {
             let current_thread_name = std::thread::current().name().unwrap().to_string();
 
             while let Ok(solana_scheduler::ExecutablePayload(mut ee)) = scheduled_ee_receiver.recv() {
@@ -1329,7 +1329,7 @@ impl Default for Scheduler {
         let errors = Arc::new(std::sync::Mutex::new(Vec::new()));
         let errors_in_collector_thread = Arc::clone(&errors);
 
-        let error_collector_thread_handle = std::thread::Builder::new().name(format!("solErrorCol{:02}", 0)).spawn(move || {
+        let error_collector_thread_handle = std::thread::Builder::new().name(format!("solScErrCol{:02}", 0)).spawn(move || {
             while let Ok(solana_scheduler::DroppablePayload(mut ee)) = droppable_ee_receiver.recv() {
                 if ee.is_aborted() {
                     warn!(
diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 28c0176458099e..71f22f9e79df9a 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1360,7 +1360,7 @@ impl ScheduleStage {
             (to_next_stage, None)
         } else {
             let h = std::thread::Builder::new()
-                .name("solReaper".to_string())
+                .name("solScReaper".to_string())
                 .spawn(move || {
                     #[derive(Clone, Copy, Debug)]
                     struct NotAtTopOfScheduleThread;
@@ -1390,7 +1390,7 @@ impl ScheduleStage {
         let indexer_handles = (0..indexer_count).map(|thx| {
             let task_receiver = task_receiver.clone();
             std::thread::Builder::new()
-                .name(format!("solIndexer{:02}", thx))
+                .name(format!("solScIdxer{:02}", thx))
                 .spawn(move || {
                     while let Ok((task, ll)) = task_receiver.recv() {
                         for lock_attempt in ll {

From 11d6f6570c44ba02d9fd0fbc8d7bf3b48961ea52 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Wed, 7 Sep 2022 16:03:25 +0900
Subject: [PATCH 2235/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 71f22f9e79df9a..6ebca777b75c14 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1453,7 +1453,7 @@ impl ScheduleStage {
                     < max_executing_queue_count
                 {
                     trace!("schedule_once (from: {}, to: {}, runnnable: {}, contended: {}, (immediate+provisional)/max: ({}+{})/{}) active from contended: {} stuck: {} completed: {}!", from_prev.len(), to_execute_substage.len(), runnable_queue.task_count(), contended_count, executing_queue_count, provisioning_tracker_count, max_executing_queue_count, address_book.uncontended_task_ids.len(), address_book.stuck_tasks.len(), completed_count);
-                    if start.elapsed() > std::time::Duration::from_millis(1000) {
+                    if start.elapsed() > std::time::Duration::from_millis(150) {
                         start = std::time::Instant::now();
                         info!("schedule_once:interval (from: {}, to: {}, runnnable: {}, contended: {}, (immediate+provisional)/max: ({}+{})/{}) active from contended: {} stuck: {} completed: {}!", from_prev.len(), to_execute_substage.len(), runnable_queue.task_count(), contended_count, executing_queue_count, provisioning_tracker_count, max_executing_queue_count, address_book.uncontended_task_ids.len(), address_book.stuck_tasks.len(), completed_count);
                     }

From 3c9c6cd9b4bed797e03410d4473015570e74b37b Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Wed, 7 Sep 2022 16:10:40 +0900
Subject: [PATCH 2236/3199] save

---
 scheduler/src/lib.rs | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 6ebca777b75c14..9dee7e3c09fa51 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1409,7 +1409,7 @@ impl ScheduleStage {
         }).collect::<Vec<_>>();
         let mut start = std::time::Instant::now();
 
-        let (mut from_disconnected, mut from_exec_disconnected, mut no_more_work) = Default::default();
+        let (mut from_disconnected, mut from_exec_disconnected, mut no_more_work, mut no_more_work2) = Default::default();
         loop {
             crossbeam_channel::select! {
                recv(from_exec) -> maybe_from_exec => {
@@ -1419,6 +1419,7 @@ impl ScheduleStage {
                        if from_disconnected {
                            break;
                        } else {
+                           info!("flushing1..: {} {} {} {} {} {}", from_disconnected, from_exec_disconnected, runnable_queue.task_count(), contended_count,  executing_queue_count, provisioning_tracker_count)
                            continue;
                        }
                    }
@@ -1436,6 +1437,7 @@ impl ScheduleStage {
                        if from_exec_disconnected || no_more_work {
                            break;
                        } else {
+                           info!("flushing2..: {} {} {} {} {} {}", from_disconnected, from_exec_disconnected, runnable_queue.task_count(), contended_count,  executing_queue_count, provisioning_tracker_count)
                            continue;
                        }
                    }

From c0a465f1537c8f529c99291f297c49ac3db8b54d Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Wed, 7 Sep 2022 16:11:00 +0900
Subject: [PATCH 2237/3199] save

---
 scheduler/src/lib.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 9dee7e3c09fa51..ec94b5d689246e 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1419,7 +1419,7 @@ impl ScheduleStage {
                        if from_disconnected {
                            break;
                        } else {
-                           info!("flushing1..: {} {} {} {} {} {}", from_disconnected, from_exec_disconnected, runnable_queue.task_count(), contended_count,  executing_queue_count, provisioning_tracker_count)
+                           info!("flushing1..: {} {} {} {} {} {}", from_disconnected, from_exec_disconnected, runnable_queue.task_count(), contended_count,  executing_queue_count, provisioning_tracker_count);
                            continue;
                        }
                    }
@@ -1437,7 +1437,7 @@ impl ScheduleStage {
                        if from_exec_disconnected || no_more_work {
                            break;
                        } else {
-                           info!("flushing2..: {} {} {} {} {} {}", from_disconnected, from_exec_disconnected, runnable_queue.task_count(), contended_count,  executing_queue_count, provisioning_tracker_count)
+                           info!("flushing2..: {} {} {} {} {} {}", from_disconnected, from_exec_disconnected, runnable_queue.task_count(), contended_count,  executing_queue_count, provisioning_tracker_count);
                            continue;
                        }
                    }

From c4f520930921f5ce4245e9ba9ab6ad3d14ef10c2 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Wed, 7 Sep 2022 16:11:25 +0900
Subject: [PATCH 2238/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index ec94b5d689246e..8712822d65ac56 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1409,7 +1409,7 @@ impl ScheduleStage {
         }).collect::<Vec<_>>();
         let mut start = std::time::Instant::now();
 
-        let (mut from_disconnected, mut from_exec_disconnected, mut no_more_work, mut no_more_work2) = Default::default();
+        let (mut from_disconnected, mut from_exec_disconnected, mut no_more_work) = Default::default();
         loop {
             crossbeam_channel::select! {
                recv(from_exec) -> maybe_from_exec => {

From d9106402fd4f31b9a7ad928d9aa0b5d0f94b797e Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Wed, 7 Sep 2022 16:18:05 +0900
Subject: [PATCH 2239/3199] save

---
 scheduler/src/lib.rs | 24 +++++++++---------------
 1 file changed, 9 insertions(+), 15 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 8712822d65ac56..7294464b0b65e5 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1413,36 +1413,30 @@ impl ScheduleStage {
         loop {
             crossbeam_channel::select! {
                recv(from_exec) -> maybe_from_exec => {
-                   if maybe_from_exec.is_err() {
+                   if let Ok((mut processed_execution_environment)) = maybe_from_exec. {
+                       executing_queue_count = executing_queue_count.checked_sub(1).unwrap();
+                       completed_count = completed_count.checked_add(1).unwrap();
+                       Self::commit_completed_execution(ast, &mut processed_execution_environment, address_book, &mut commit_clock, &mut provisioning_tracker_count);
+                       to_next_stage.send(DroppablePayload(processed_execution_environment)).unwrap();
+                   } else {
                        assert_eq!(from_exec.len(), 0);
                        from_exec_disconnected |= true;
                        if from_disconnected {
                            break;
-                       } else {
-                           info!("flushing1..: {} {} {} {} {} {}", from_disconnected, from_exec_disconnected, runnable_queue.task_count(), contended_count,  executing_queue_count, provisioning_tracker_count);
-                           continue;
                        }
                    }
-                   let mut processed_execution_environment = maybe_from_exec.unwrap().0;
-                    executing_queue_count = executing_queue_count.checked_sub(1).unwrap();
-                    completed_count = completed_count.checked_add(1).unwrap();
-                    Self::commit_completed_execution(ast, &mut processed_execution_environment, address_book, &mut commit_clock, &mut provisioning_tracker_count);
-                    to_next_stage.send(DroppablePayload(processed_execution_environment)).unwrap();
                }
                recv(from_prev) -> maybe_from => {
-                   if maybe_from.is_err() {
+                   if let Ok((Task)) = maybe_from {
+                       Self::register_runnable_task(task, runnable_queue, &mut sequence_time);
+                   } else {
                        assert_eq!(from_prev.len(), 0);
                        from_disconnected |= true;
                        no_more_work |= runnable_queue.task_count() + contended_count + executing_queue_count + provisioning_tracker_count == 0;
                        if from_exec_disconnected || no_more_work {
                            break;
-                       } else {
-                           info!("flushing2..: {} {} {} {} {} {}", from_disconnected, from_exec_disconnected, runnable_queue.task_count(), contended_count,  executing_queue_count, provisioning_tracker_count);
-                           continue;
                        }
                    }
-                   let task = maybe_from.unwrap().0;
-                   Self::register_runnable_task(task, runnable_queue, &mut sequence_time);
                }
             }
 

From 510e07e4b218e1400c875f224c7a649750a64111 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Wed, 7 Sep 2022 16:18:33 +0900
Subject: [PATCH 2240/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 7294464b0b65e5..c81610428eeebe 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1413,7 +1413,7 @@ impl ScheduleStage {
         loop {
             crossbeam_channel::select! {
                recv(from_exec) -> maybe_from_exec => {
-                   if let Ok((mut processed_execution_environment)) = maybe_from_exec. {
+                   if let Ok(UnlockablePayload(mut processed_execution_environment)) = maybe_from_exec {
                        executing_queue_count = executing_queue_count.checked_sub(1).unwrap();
                        completed_count = completed_count.checked_add(1).unwrap();
                        Self::commit_completed_execution(ast, &mut processed_execution_environment, address_book, &mut commit_clock, &mut provisioning_tracker_count);

From 144caaa8bdece3f50e7a06e84fc56e5cfecb7870 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Wed, 7 Sep 2022 16:18:53 +0900
Subject: [PATCH 2241/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index c81610428eeebe..6388756ee0555c 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1427,7 +1427,7 @@ impl ScheduleStage {
                    }
                }
                recv(from_prev) -> maybe_from => {
-                   if let Ok((Task)) = maybe_from {
+                   if let Ok(SchedulablePayload(task)) = maybe_from {
                        Self::register_runnable_task(task, runnable_queue, &mut sequence_time);
                    } else {
                        assert_eq!(from_prev.len(), 0);

From 9826236629fcd94c73313d9a9df62deea9dc2c49 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Wed, 7 Sep 2022 16:20:40 +0900
Subject: [PATCH 2242/3199] save

---
 scheduler/src/lib.rs | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 6388756ee0555c..7101db6db9f76d 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1421,6 +1421,7 @@ impl ScheduleStage {
                    } else {
                        assert_eq!(from_exec.len(), 0);
                        from_exec_disconnected |= true;
+                       info!("flushing1..: {} {} {} {} {} {}", from_disconnected, from_exec_disconnected, runnable_queue.task_count(), contended_count,  executing_queue_count, provisioning_tracker_count);
                        if from_disconnected {
                            break;
                        }
@@ -1433,6 +1434,7 @@ impl ScheduleStage {
                        assert_eq!(from_prev.len(), 0);
                        from_disconnected |= true;
                        no_more_work |= runnable_queue.task_count() + contended_count + executing_queue_count + provisioning_tracker_count == 0;
+                       info!("flushing2..: {} {} {} {} {} {}", from_disconnected, from_exec_disconnected, runnable_queue.task_count(), contended_count,  executing_queue_count, provisioning_tracker_count);
                        if from_exec_disconnected || no_more_work {
                            break;
                        }

From 262c3e24a908b310499743e22b930cb77da007f1 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Wed, 7 Sep 2022 16:39:28 +0900
Subject: [PATCH 2243/3199] save

---
 scheduler/src/lib.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 7101db6db9f76d..e60112cf17887c 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1450,10 +1450,10 @@ impl ScheduleStage {
                 while (executing_queue_count + provisioning_tracker_count)
                     < max_executing_queue_count
                 {
-                    trace!("schedule_once (from: {}, to: {}, runnnable: {}, contended: {}, (immediate+provisional)/max: ({}+{})/{}) active from contended: {} stuck: {} completed: {}!", from_prev.len(), to_execute_substage.len(), runnable_queue.task_count(), contended_count, executing_queue_count, provisioning_tracker_count, max_executing_queue_count, address_book.uncontended_task_ids.len(), address_book.stuck_tasks.len(), completed_count);
+                    trace!("schedule_once id_{:016x} (from: {}, to: {}, runnnable: {}, contended: {}, (immediate+provisional)/max: ({}+{})/{}) active from contended: {} stuck: {} completed: {}!", random_id, from_prev.len(), to_execute_substage.len(), runnable_queue.task_count(), contended_count, executing_queue_count, provisioning_tracker_count, max_executing_queue_count, address_book.uncontended_task_ids.len(), address_book.stuck_tasks.len(), completed_count);
                     if start.elapsed() > std::time::Duration::from_millis(150) {
                         start = std::time::Instant::now();
-                        info!("schedule_once:interval (from: {}, to: {}, runnnable: {}, contended: {}, (immediate+provisional)/max: ({}+{})/{}) active from contended: {} stuck: {} completed: {}!", from_prev.len(), to_execute_substage.len(), runnable_queue.task_count(), contended_count, executing_queue_count, provisioning_tracker_count, max_executing_queue_count, address_book.uncontended_task_ids.len(), address_book.stuck_tasks.len(), completed_count);
+                        info!("schedule_once:interval id_{:016x} (from: {}, to: {}, runnnable: {}, contended: {}, (immediate+provisional)/max: ({}+{})/{}) active from contended: {} stuck: {} completed: {}!", random_id, from_prev.len(), to_execute_substage.len(), runnable_queue.task_count(), contended_count, executing_queue_count, provisioning_tracker_count, max_executing_queue_count, address_book.uncontended_task_ids.len(), address_book.stuck_tasks.len(), completed_count);
                     }
                     let prefer_immediate = provisioning_tracker_count / 4 > executing_queue_count;
                     if let Some(ee) = Self::schedule_next_execution(

From afc6618e5955218baa2db1a97bbe2e058879f8d1 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Wed, 7 Sep 2022 16:47:38 +0900
Subject: [PATCH 2244/3199] save

---
 scheduler/src/lib.rs | 10 ++++------
 1 file changed, 4 insertions(+), 6 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index e60112cf17887c..fa94c03a31e2d4 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1029,7 +1029,8 @@ impl ScheduleStage {
                         */
 
                         if from_runnable {
-                            continue; // continue to prefer depleting the possibly-non-empty runnable queue
+                            // continue; // continue to prefer depleting the possibly-non-empty runnable queue
+                            break;
                         } else if task_source == TaskSource::Stuck {
                             // need to bail out immediately to avoid going to infinite loop of re-processing
                             // the struck task again.
@@ -1447,9 +1448,8 @@ impl ScheduleStage {
             let (mut from_len, mut from_exec_len) = (0, 0);
 
             loop {
-                while (executing_queue_count + provisioning_tracker_count)
-                    < max_executing_queue_count
-                {
+                let executing_like_count = executing_queue_count + provisioning_tracker_count;
+                if executing_like_count < max_executing_queue_count {
                     trace!("schedule_once id_{:016x} (from: {}, to: {}, runnnable: {}, contended: {}, (immediate+provisional)/max: ({}+{})/{}) active from contended: {} stuck: {} completed: {}!", random_id, from_prev.len(), to_execute_substage.len(), runnable_queue.task_count(), contended_count, executing_queue_count, provisioning_tracker_count, max_executing_queue_count, address_book.uncontended_task_ids.len(), address_book.stuck_tasks.len(), completed_count);
                     if start.elapsed() > std::time::Duration::from_millis(150) {
                         start = std::time::Instant::now();
@@ -1470,8 +1470,6 @@ impl ScheduleStage {
                     ) {
                         executing_queue_count = executing_queue_count.checked_add(1).unwrap();
                         to_execute_substage.send(ExecutablePayload(ee)).unwrap();
-                    } else {
-                        break;
                     }
                 }
                 //break;

From a14f0ffb11d1fb1e25ebf3900f547ea993734293 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Wed, 7 Sep 2022 16:54:44 +0900
Subject: [PATCH 2245/3199] save

---
 scheduler/src/lib.rs | 1 +
 1 file changed, 1 insertion(+)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index fa94c03a31e2d4..109f055fba155e 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1433,6 +1433,7 @@ impl ScheduleStage {
                        Self::register_runnable_task(task, runnable_queue, &mut sequence_time);
                    } else {
                        assert_eq!(from_prev.len(), 0);
+                       from_prev = never();
                        from_disconnected |= true;
                        no_more_work |= runnable_queue.task_count() + contended_count + executing_queue_count + provisioning_tracker_count == 0;
                        info!("flushing2..: {} {} {} {} {} {}", from_disconnected, from_exec_disconnected, runnable_queue.task_count(), contended_count,  executing_queue_count, provisioning_tracker_count);

From 921131a077424a831a6d473c9b47278574c2c881 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Wed, 7 Sep 2022 16:55:00 +0900
Subject: [PATCH 2246/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 109f055fba155e..2adf392442b16d 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1433,7 +1433,7 @@ impl ScheduleStage {
                        Self::register_runnable_task(task, runnable_queue, &mut sequence_time);
                    } else {
                        assert_eq!(from_prev.len(), 0);
-                       from_prev = never();
+                       from_prev = crossbeam_channel::never();
                        from_disconnected |= true;
                        no_more_work |= runnable_queue.task_count() + contended_count + executing_queue_count + provisioning_tracker_count == 0;
                        info!("flushing2..: {} {} {} {} {} {}", from_disconnected, from_exec_disconnected, runnable_queue.task_count(), contended_count,  executing_queue_count, provisioning_tracker_count);

From 0fcf5e3a30514eb444da22b5a08ce6c73255b03a Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Wed, 7 Sep 2022 16:55:24 +0900
Subject: [PATCH 2247/3199] save

---
 scheduler/src/lib.rs | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 2adf392442b16d..4e00c138dc2c4e 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1409,6 +1409,7 @@ impl ScheduleStage {
                 .unwrap()
         }).collect::<Vec<_>>();
         let mut start = std::time::Instant::now();
+        let never = crossbeam_channel::never();
 
         let (mut from_disconnected, mut from_exec_disconnected, mut no_more_work) = Default::default();
         loop {
@@ -1433,7 +1434,7 @@ impl ScheduleStage {
                        Self::register_runnable_task(task, runnable_queue, &mut sequence_time);
                    } else {
                        assert_eq!(from_prev.len(), 0);
-                       from_prev = crossbeam_channel::never();
+                       from_prev = &never;
                        from_disconnected |= true;
                        no_more_work |= runnable_queue.task_count() + contended_count + executing_queue_count + provisioning_tracker_count == 0;
                        info!("flushing2..: {} {} {} {} {} {}", from_disconnected, from_exec_disconnected, runnable_queue.task_count(), contended_count,  executing_queue_count, provisioning_tracker_count);

From 50606d839f6b56eab3cf8b2dd77aa459125db512 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Wed, 7 Sep 2022 16:56:08 +0900
Subject: [PATCH 2248/3199] save

---
 scheduler/src/lib.rs | 1 +
 1 file changed, 1 insertion(+)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 4e00c138dc2c4e..29eb0afbb5311a 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1515,6 +1515,7 @@ impl ScheduleStage {
                 }
             }
         }
+        drop(from_prev);
         drop(to_next_stage);
         drop(ee_sender);
         drop(task_sender);

From 0a90c8712279259699a5d12ace160d645f418d9c Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Wed, 7 Sep 2022 16:57:17 +0900
Subject: [PATCH 2249/3199] save

---
 scheduler/src/lib.rs | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 29eb0afbb5311a..d05fa6d9d6da2e 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1341,6 +1341,7 @@ impl ScheduleStage {
         to_execute_substage: &crossbeam_channel::Sender<ExecutablePayload>,
         from_exec: &crossbeam_channel::Receiver<UnlockablePayload>,
         maybe_to_next_stage: Option<&crossbeam_channel::Sender<DroppablePayload>>, // assume nonblocking
+        never: usize,
     ) {
         let random_id = rand::thread_rng().gen::<u64>();
         info!("schedule_once:initial id_{:016x}", random_id);
@@ -1409,7 +1410,6 @@ impl ScheduleStage {
                 .unwrap()
         }).collect::<Vec<_>>();
         let mut start = std::time::Instant::now();
-        let never = crossbeam_channel::never();
 
         let (mut from_disconnected, mut from_exec_disconnected, mut no_more_work) = Default::default();
         loop {
@@ -1544,6 +1544,8 @@ impl ScheduleStage {
         struct AtTopOfScheduleThread;
         unsafe impl AtScheduleThread for AtTopOfScheduleThread {}
 
+        let never = crossbeam_channel::never();
+
         Self::_run::<AtTopOfScheduleThread>(
             AtTopOfScheduleThread,
             max_executing_queue_count,
@@ -1553,6 +1555,7 @@ impl ScheduleStage {
             to_execute_substage,
             from_execute_substage,
             maybe_to_next_stage,
+            never,
         )
     }
 }

From 296ea727277ff313cb046f5e576ce70cde5f66ea Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Wed, 7 Sep 2022 16:57:31 +0900
Subject: [PATCH 2250/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index d05fa6d9d6da2e..1cf98f0b6b8578 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1341,7 +1341,7 @@ impl ScheduleStage {
         to_execute_substage: &crossbeam_channel::Sender<ExecutablePayload>,
         from_exec: &crossbeam_channel::Receiver<UnlockablePayload>,
         maybe_to_next_stage: Option<&crossbeam_channel::Sender<DroppablePayload>>, // assume nonblocking
-        never: usize,
+        never: crossbeam_channel::Receiver,
     ) {
         let random_id = rand::thread_rng().gen::<u64>();
         info!("schedule_once:initial id_{:016x}", random_id);

From 8c86f35e3bd10d442c596e2389e90c96af5baf87 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Wed, 7 Sep 2022 16:57:50 +0900
Subject: [PATCH 2251/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 1cf98f0b6b8578..e78b65e8547899 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1341,7 +1341,7 @@ impl ScheduleStage {
         to_execute_substage: &crossbeam_channel::Sender<ExecutablePayload>,
         from_exec: &crossbeam_channel::Receiver<UnlockablePayload>,
         maybe_to_next_stage: Option<&crossbeam_channel::Sender<DroppablePayload>>, // assume nonblocking
-        never: crossbeam_channel::Receiver,
+        never: &crossbeam_channel::Receiver<SchedulablePayload>,
     ) {
         let random_id = rand::thread_rng().gen::<u64>();
         info!("schedule_once:initial id_{:016x}", random_id);

From 2ba06f61bc8c4af2b0565cecb8f2e513fe384fd5 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Wed, 7 Sep 2022 16:58:03 +0900
Subject: [PATCH 2252/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index e78b65e8547899..9303990c2af69e 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1555,7 +1555,7 @@ impl ScheduleStage {
             to_execute_substage,
             from_execute_substage,
             maybe_to_next_stage,
-            never,
+            &never,
         )
     }
 }

From e7c409fbfee8a83dce16b1e7df99bb0018f073b1 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Wed, 7 Sep 2022 16:58:32 +0900
Subject: [PATCH 2253/3199] save

---
 scheduler/src/lib.rs | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 9303990c2af69e..d97c7841562724 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1332,16 +1332,16 @@ impl ScheduleStage {
         Self::push_to_runnable_queue(weighted_tx, runnable_queue)
     }
 
-    fn _run<AST: AtScheduleThread>(
+    fn _run<'a, AST: AtScheduleThread>(
         ast: AST,
         max_executing_queue_count: usize,
         runnable_queue: &mut TaskQueue,
         address_book: &mut AddressBook,
-        from_prev: &crossbeam_channel::Receiver<SchedulablePayload>,
+        from_prev: &'a crossbeam_channel::Receiver<SchedulablePayload>,
         to_execute_substage: &crossbeam_channel::Sender<ExecutablePayload>,
         from_exec: &crossbeam_channel::Receiver<UnlockablePayload>,
         maybe_to_next_stage: Option<&crossbeam_channel::Sender<DroppablePayload>>, // assume nonblocking
-        never: &crossbeam_channel::Receiver<SchedulablePayload>,
+        never: &'a crossbeam_channel::Receiver<SchedulablePayload>,
     ) {
         let random_id = rand::thread_rng().gen::<u64>();
         info!("schedule_once:initial id_{:016x}", random_id);

From 498d5b0b8c35c3b215361160bb6877295d9a1bf2 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Wed, 7 Sep 2022 16:58:42 +0900
Subject: [PATCH 2254/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index d97c7841562724..d7e3d3cc163163 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1337,7 +1337,7 @@ impl ScheduleStage {
         max_executing_queue_count: usize,
         runnable_queue: &mut TaskQueue,
         address_book: &mut AddressBook,
-        from_prev: &'a crossbeam_channel::Receiver<SchedulablePayload>,
+        mut from_prev: &'a crossbeam_channel::Receiver<SchedulablePayload>,
         to_execute_substage: &crossbeam_channel::Sender<ExecutablePayload>,
         from_exec: &crossbeam_channel::Receiver<UnlockablePayload>,
         maybe_to_next_stage: Option<&crossbeam_channel::Sender<DroppablePayload>>, // assume nonblocking

From 498f72a112c8ec9e9ca9728a34853cf6beb93f43 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Wed, 7 Sep 2022 16:59:21 +0900
Subject: [PATCH 2255/3199] save

---
 scheduler/src/lib.rs | 1 +
 1 file changed, 1 insertion(+)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index d7e3d3cc163163..f7270fbcb42d42 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1435,6 +1435,7 @@ impl ScheduleStage {
                    } else {
                        assert_eq!(from_prev.len(), 0);
                        from_prev = &never;
+                       assert_eq!(from_prev.len(), 0);
                        from_disconnected |= true;
                        no_more_work |= runnable_queue.task_count() + contended_count + executing_queue_count + provisioning_tracker_count == 0;
                        info!("flushing2..: {} {} {} {} {} {}", from_disconnected, from_exec_disconnected, runnable_queue.task_count(), contended_count,  executing_queue_count, provisioning_tracker_count);

From 12c080e1671755012991467b24204957aa6f1578 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Wed, 7 Sep 2022 17:21:55 +0900
Subject: [PATCH 2256/3199] save

---
 scheduler/src/lib.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index f7270fbcb42d42..918c9020eb3449 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1434,8 +1434,8 @@ impl ScheduleStage {
                        Self::register_runnable_task(task, runnable_queue, &mut sequence_time);
                    } else {
                        assert_eq!(from_prev.len(), 0);
-                       from_prev = &never;
-                       assert_eq!(from_prev.len(), 0);
+                       //from_prev = &never;
+                       //assert_eq!(from_prev.len(), 0);
                        from_disconnected |= true;
                        no_more_work |= runnable_queue.task_count() + contended_count + executing_queue_count + provisioning_tracker_count == 0;
                        info!("flushing2..: {} {} {} {} {} {}", from_disconnected, from_exec_disconnected, runnable_queue.task_count(), contended_count,  executing_queue_count, provisioning_tracker_count);

From 6eed540f883120e3a8637ae143332a122ccabfa6 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Wed, 7 Sep 2022 17:23:06 +0900
Subject: [PATCH 2257/3199] save

---
 scheduler/src/lib.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 918c9020eb3449..699ba958b44bf4 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1423,7 +1423,7 @@ impl ScheduleStage {
                    } else {
                        assert_eq!(from_exec.len(), 0);
                        from_exec_disconnected |= true;
-                       info!("flushing1..: {} {} {} {} {} {}", from_disconnected, from_exec_disconnected, runnable_queue.task_count(), contended_count,  executing_queue_count, provisioning_tracker_count);
+                       //info!("flushing1..: {} {} {} {} {} {}", from_disconnected, from_exec_disconnected, runnable_queue.task_count(), contended_count,  executing_queue_count, provisioning_tracker_count);
                        if from_disconnected {
                            break;
                        }
@@ -1438,7 +1438,7 @@ impl ScheduleStage {
                        //assert_eq!(from_prev.len(), 0);
                        from_disconnected |= true;
                        no_more_work |= runnable_queue.task_count() + contended_count + executing_queue_count + provisioning_tracker_count == 0;
-                       info!("flushing2..: {} {} {} {} {} {}", from_disconnected, from_exec_disconnected, runnable_queue.task_count(), contended_count,  executing_queue_count, provisioning_tracker_count);
+                       //info!("flushing2..: {} {} {} {} {} {}", from_disconnected, from_exec_disconnected, runnable_queue.task_count(), contended_count,  executing_queue_count, provisioning_tracker_count);
                        if from_exec_disconnected || no_more_work {
                            break;
                        }

From bc6f9f35f3712cb80f719a932c0425b83955416a Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Wed, 7 Sep 2022 17:27:51 +0900
Subject: [PATCH 2258/3199] save

---
 scheduler/src/lib.rs | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 699ba958b44bf4..da3cb95f970608 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1413,6 +1413,7 @@ impl ScheduleStage {
 
         let (mut from_disconnected, mut from_exec_disconnected, mut no_more_work) = Default::default();
         loop {
+            if !from_disconnected || executing_queue_count > 0 {
             crossbeam_channel::select! {
                recv(from_exec) -> maybe_from_exec => {
                    if let Ok(UnlockablePayload(mut processed_execution_environment)) = maybe_from_exec {
@@ -1423,7 +1424,7 @@ impl ScheduleStage {
                    } else {
                        assert_eq!(from_exec.len(), 0);
                        from_exec_disconnected |= true;
-                       //info!("flushing1..: {} {} {} {} {} {}", from_disconnected, from_exec_disconnected, runnable_queue.task_count(), contended_count,  executing_queue_count, provisioning_tracker_count);
+                       info!("flushing1..: {} {} {} {} {} {}", from_disconnected, from_exec_disconnected, runnable_queue.task_count(), contended_count,  executing_queue_count, provisioning_tracker_count);
                        if from_disconnected {
                            break;
                        }
@@ -1434,17 +1435,18 @@ impl ScheduleStage {
                        Self::register_runnable_task(task, runnable_queue, &mut sequence_time);
                    } else {
                        assert_eq!(from_prev.len(), 0);
-                       //from_prev = &never;
-                       //assert_eq!(from_prev.len(), 0);
+                       from_prev = &never;
+                       assert_eq!(from_prev.len(), 0);
                        from_disconnected |= true;
                        no_more_work |= runnable_queue.task_count() + contended_count + executing_queue_count + provisioning_tracker_count == 0;
-                       //info!("flushing2..: {} {} {} {} {} {}", from_disconnected, from_exec_disconnected, runnable_queue.task_count(), contended_count,  executing_queue_count, provisioning_tracker_count);
+                       info!("flushing2..: {} {} {} {} {} {}", from_disconnected, from_exec_disconnected, runnable_queue.task_count(), contended_count,  executing_queue_count, provisioning_tracker_count);
                        if from_exec_disconnected || no_more_work {
                            break;
                        }
                    }
                }
             }
+            }
 
             let mut first_iteration = true;
             let (mut empty_from, mut empty_from_exec) = (false, false);

From e22d341dfb32a2899a9c92058d2371dc0cc7799b Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Wed, 7 Sep 2022 17:28:34 +0900
Subject: [PATCH 2259/3199] save

---
 scheduler/src/lib.rs | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index da3cb95f970608..fd4ed42c08eae7 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1411,7 +1411,8 @@ impl ScheduleStage {
         }).collect::<Vec<_>>();
         let mut start = std::time::Instant::now();
 
-        let (mut from_disconnected, mut from_exec_disconnected, mut no_more_work) = Default::default();
+        let (mut from_disconnected, mut from_exec_disconnected, mut no_more_work): (bool, bool, bool) = Default::default();
+
         loop {
             if !from_disconnected || executing_queue_count > 0 {
             crossbeam_channel::select! {

From f3ca876fd2ef7f6f39b497f4f1d792e55b30fbd0 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Wed, 7 Sep 2022 17:35:45 +0900
Subject: [PATCH 2260/3199] save

---
 scheduler/src/lib.rs | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index fd4ed42c08eae7..8ea6cc5d6a9a15 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1439,15 +1439,15 @@ impl ScheduleStage {
                        from_prev = &never;
                        assert_eq!(from_prev.len(), 0);
                        from_disconnected |= true;
-                       no_more_work |= runnable_queue.task_count() + contended_count + executing_queue_count + provisioning_tracker_count == 0;
                        info!("flushing2..: {} {} {} {} {} {}", from_disconnected, from_exec_disconnected, runnable_queue.task_count(), contended_count,  executing_queue_count, provisioning_tracker_count);
-                       if from_exec_disconnected || no_more_work {
-                           break;
-                       }
                    }
                }
             }
             }
+           no_more_work |= runnable_queue.task_count() + contended_count + executing_queue_count + provisioning_tracker_count == 0;
+           if from_disconnected && (from_exec_disconnected || no_more_work) {
+               break;
+           }
 
             let mut first_iteration = true;
             let (mut empty_from, mut empty_from_exec) = (false, false);

From 626e1618924d5650c23fadab3a9f66fcd926e72d Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Wed, 7 Sep 2022 17:51:06 +0900
Subject: [PATCH 2261/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 8ea6cc5d6a9a15..ccd46254866f76 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1444,7 +1444,7 @@ impl ScheduleStage {
                }
             }
             }
-           no_more_work |= runnable_queue.task_count() + contended_count + executing_queue_count + provisioning_tracker_count == 0;
+           no_more_work = from_disconnected && runnable_queue.task_count() + contended_count + executing_queue_count + provisioning_tracker_count == 0;
            if from_disconnected && (from_exec_disconnected || no_more_work) {
                break;
            }

From 6a375d7274d9fe32193c5570230f5238fb475eca Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Wed, 7 Sep 2022 18:03:58 +0900
Subject: [PATCH 2262/3199] save

---
 scheduler/src/lib.rs | 24 +++++++-----------------
 1 file changed, 7 insertions(+), 17 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index ccd46254866f76..fa94c03a31e2d4 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1332,16 +1332,15 @@ impl ScheduleStage {
         Self::push_to_runnable_queue(weighted_tx, runnable_queue)
     }
 
-    fn _run<'a, AST: AtScheduleThread>(
+    fn _run<AST: AtScheduleThread>(
         ast: AST,
         max_executing_queue_count: usize,
         runnable_queue: &mut TaskQueue,
         address_book: &mut AddressBook,
-        mut from_prev: &'a crossbeam_channel::Receiver<SchedulablePayload>,
+        from_prev: &crossbeam_channel::Receiver<SchedulablePayload>,
         to_execute_substage: &crossbeam_channel::Sender<ExecutablePayload>,
         from_exec: &crossbeam_channel::Receiver<UnlockablePayload>,
         maybe_to_next_stage: Option<&crossbeam_channel::Sender<DroppablePayload>>, // assume nonblocking
-        never: &'a crossbeam_channel::Receiver<SchedulablePayload>,
     ) {
         let random_id = rand::thread_rng().gen::<u64>();
         info!("schedule_once:initial id_{:016x}", random_id);
@@ -1411,10 +1410,8 @@ impl ScheduleStage {
         }).collect::<Vec<_>>();
         let mut start = std::time::Instant::now();
 
-        let (mut from_disconnected, mut from_exec_disconnected, mut no_more_work): (bool, bool, bool) = Default::default();
-
+        let (mut from_disconnected, mut from_exec_disconnected, mut no_more_work) = Default::default();
         loop {
-            if !from_disconnected || executing_queue_count > 0 {
             crossbeam_channel::select! {
                recv(from_exec) -> maybe_from_exec => {
                    if let Ok(UnlockablePayload(mut processed_execution_environment)) = maybe_from_exec {
@@ -1435,19 +1432,16 @@ impl ScheduleStage {
                    if let Ok(SchedulablePayload(task)) = maybe_from {
                        Self::register_runnable_task(task, runnable_queue, &mut sequence_time);
                    } else {
-                       assert_eq!(from_prev.len(), 0);
-                       from_prev = &never;
                        assert_eq!(from_prev.len(), 0);
                        from_disconnected |= true;
+                       no_more_work |= runnable_queue.task_count() + contended_count + executing_queue_count + provisioning_tracker_count == 0;
                        info!("flushing2..: {} {} {} {} {} {}", from_disconnected, from_exec_disconnected, runnable_queue.task_count(), contended_count,  executing_queue_count, provisioning_tracker_count);
+                       if from_exec_disconnected || no_more_work {
+                           break;
+                       }
                    }
                }
             }
-            }
-           no_more_work = from_disconnected && runnable_queue.task_count() + contended_count + executing_queue_count + provisioning_tracker_count == 0;
-           if from_disconnected && (from_exec_disconnected || no_more_work) {
-               break;
-           }
 
             let mut first_iteration = true;
             let (mut empty_from, mut empty_from_exec) = (false, false);
@@ -1519,7 +1513,6 @@ impl ScheduleStage {
                 }
             }
         }
-        drop(from_prev);
         drop(to_next_stage);
         drop(ee_sender);
         drop(task_sender);
@@ -1548,8 +1541,6 @@ impl ScheduleStage {
         struct AtTopOfScheduleThread;
         unsafe impl AtScheduleThread for AtTopOfScheduleThread {}
 
-        let never = crossbeam_channel::never();
-
         Self::_run::<AtTopOfScheduleThread>(
             AtTopOfScheduleThread,
             max_executing_queue_count,
@@ -1559,7 +1550,6 @@ impl ScheduleStage {
             to_execute_substage,
             from_execute_substage,
             maybe_to_next_stage,
-            &never,
         )
     }
 }

From 5dc8f24062287a7da9547976d0875bee9c439e6f Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Wed, 7 Sep 2022 21:28:11 +0900
Subject: [PATCH 2263/3199] save

---
 scheduler/src/lib.rs | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index fa94c03a31e2d4..ab53acc1f61885 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1449,11 +1449,11 @@ impl ScheduleStage {
 
             loop {
                 let executing_like_count = executing_queue_count + provisioning_tracker_count;
-                if executing_like_count < max_executing_queue_count {
-                    trace!("schedule_once id_{:016x} (from: {}, to: {}, runnnable: {}, contended: {}, (immediate+provisional)/max: ({}+{})/{}) active from contended: {} stuck: {} completed: {}!", random_id, from_prev.len(), to_execute_substage.len(), runnable_queue.task_count(), contended_count, executing_queue_count, provisioning_tracker_count, max_executing_queue_count, address_book.uncontended_task_ids.len(), address_book.stuck_tasks.len(), completed_count);
+                while executing_like_count < max_executing_queue_count {
+                    trace!("schedule_once id_{:016x} (prev: {}, exec: ({}|{}), runnnable: {}, contended: {}, (immediate+provisional)/max: ({}+{})/{}) active from contended: {} stuck: {} completed: {}!", random_id, from_prev.len(), to_execute_substage.len(), from_exec.len(), runnable_queue.task_count(), contended_count, executing_queue_count, provisioning_tracker_count, max_executing_queue_count, address_book.uncontended_task_ids.len(), address_book.stuck_tasks.len(), completed_count);
                     if start.elapsed() > std::time::Duration::from_millis(150) {
                         start = std::time::Instant::now();
-                        info!("schedule_once:interval id_{:016x} (from: {}, to: {}, runnnable: {}, contended: {}, (immediate+provisional)/max: ({}+{})/{}) active from contended: {} stuck: {} completed: {}!", random_id, from_prev.len(), to_execute_substage.len(), runnable_queue.task_count(), contended_count, executing_queue_count, provisioning_tracker_count, max_executing_queue_count, address_book.uncontended_task_ids.len(), address_book.stuck_tasks.len(), completed_count);
+                        info!("schedule_once:interval id_{:016x} (prev: {}, exec: ({}|{}), runnnable: {}, contended: {}, (immediate+provisional)/max: ({}+{})/{}) active from contended: {} stuck: {} completed: {}!", random_id, from_prev.len(), to_execute_substage.len(), from_exec.len(), runnable_queue.task_count(), contended_count, executing_queue_count, provisioning_tracker_count, max_executing_queue_count, address_book.uncontended_task_ids.len(), address_book.stuck_tasks.len(), completed_count);
                     }
                     let prefer_immediate = provisioning_tracker_count / 4 > executing_queue_count;
                     if let Some(ee) = Self::schedule_next_execution(
@@ -1470,6 +1470,8 @@ impl ScheduleStage {
                     ) {
                         executing_queue_count = executing_queue_count.checked_add(1).unwrap();
                         to_execute_substage.send(ExecutablePayload(ee)).unwrap();
+                    } else {
+                        break;
                     }
                 }
                 //break;
@@ -1525,7 +1527,7 @@ impl ScheduleStage {
             indexer_handle.join().unwrap().unwrap();
         }
 
-        info!("schedule_once:final id_{:016x} (from_disconnected: {}, from_exec_disconnected: {}, no_more_work: {}) (from: {}, to: {}, runnnable: {}, contended: {}, (immediate+provisional)/max: ({}+{})/{}) active from contended: {} stuck: {} completed: {}!", random_id, from_disconnected, from_exec_disconnected, no_more_work, from_prev.len(), to_execute_substage.len(), runnable_queue.task_count(), contended_count, executing_queue_count, provisioning_tracker_count, max_executing_queue_count, address_book.uncontended_task_ids.len(), address_book.stuck_tasks.len(), completed_count);
+        info!("schedule_once:final id_{:016x} (from_disconnected: {}, from_exec_disconnected: {}, no_more_work: {}) (prev: {}, exec: ({}|{}), runnnable: {}, contended: {}, (immediate+provisional)/max: ({}+{})/{}) active from contended: {} stuck: {} completed: {}!", random_id, from_disconnected, from_exec_disconnected, no_more_work, from_prev.len(), to_execute_substage.len(), from_exec.len(), runnable_queue.task_count(), contended_count, executing_queue_count, provisioning_tracker_count, max_executing_queue_count, address_book.uncontended_task_ids.len(), address_book.stuck_tasks.len(), completed_count);
     }
 
     pub fn run(

From cbff7a82372e01e025fff67111dbda63b4468f14 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Wed, 7 Sep 2022 21:33:08 +0900
Subject: [PATCH 2264/3199] save

---
 scheduler/src/lib.rs | 35 +++++++++++++++++++----------------
 1 file changed, 19 insertions(+), 16 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index ab53acc1f61885..4d711d70f4269f 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1449,14 +1449,14 @@ impl ScheduleStage {
 
             loop {
                 let executing_like_count = executing_queue_count + provisioning_tracker_count;
-                while executing_like_count < max_executing_queue_count {
+                while executing_like_count < max_executing_queue_count && empty_from_exec {
                     trace!("schedule_once id_{:016x} (prev: {}, exec: ({}|{}), runnnable: {}, contended: {}, (immediate+provisional)/max: ({}+{})/{}) active from contended: {} stuck: {} completed: {}!", random_id, from_prev.len(), to_execute_substage.len(), from_exec.len(), runnable_queue.task_count(), contended_count, executing_queue_count, provisioning_tracker_count, max_executing_queue_count, address_book.uncontended_task_ids.len(), address_book.stuck_tasks.len(), completed_count);
                     if start.elapsed() > std::time::Duration::from_millis(150) {
                         start = std::time::Instant::now();
                         info!("schedule_once:interval id_{:016x} (prev: {}, exec: ({}|{}), runnnable: {}, contended: {}, (immediate+provisional)/max: ({}+{})/{}) active from contended: {} stuck: {} completed: {}!", random_id, from_prev.len(), to_execute_substage.len(), from_exec.len(), runnable_queue.task_count(), contended_count, executing_queue_count, provisioning_tracker_count, max_executing_queue_count, address_book.uncontended_task_ids.len(), address_book.stuck_tasks.len(), completed_count);
                     }
                     let prefer_immediate = provisioning_tracker_count / 4 > executing_queue_count;
-                    if let Some(ee) = Self::schedule_next_execution(
+                    let maybe_ee = Self::schedule_next_execution(
                         ast,
                         &task_sender,
                         runnable_queue,
@@ -1467,26 +1467,29 @@ impl ScheduleStage {
                         &mut queue_clock,
                         &mut execute_clock,
                         &mut provisioning_tracker_count,
-                    ) {
+                    );
+                    if let Some(ee) = maybe_ee {
                         executing_queue_count = executing_queue_count.checked_add(1).unwrap();
                         to_execute_substage.send(ExecutablePayload(ee)).unwrap();
-                    } else {
-                        break;
                     }
-                }
-                //break;
-                if first_iteration {
-                    first_iteration = false;
-                    (from_len, from_exec_len) = (from_prev.len(), from_exec.len());
-                } else {
-                    if empty_from {
-                        from_len = from_prev.len();
+
+                    if first_iteration {
+                        first_iteration = false;
+                        (from_len, from_exec_len) = (from_prev.len(), from_exec.len());
+                    } else {
+                        if empty_from {
+                            from_len = from_prev.len();
+                        }
+                        if empty_from_exec {
+                            from_exec_len = from_exec.len();
+                        }
                     }
-                    if empty_from_exec {
-                        from_exec_len = from_exec.len();
+                    (empty_from, empty_from_exec) = (from_len == 0, from_exec_len == 0);
+
+                    if maybe_ee.is_none() {
+                        break;
                     }
                 }
-                (empty_from, empty_from_exec) = (from_len == 0, from_exec_len == 0);
 
                 if empty_from && empty_from_exec {
                     break;

From e59ab9c379d1fa11e2c0e3a53c3bd5b01cb0500b Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Wed, 7 Sep 2022 21:34:35 +0900
Subject: [PATCH 2265/3199] save

---
 scheduler/src/lib.rs | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 4d711d70f4269f..6bc82aa27ec58b 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1468,9 +1468,13 @@ impl ScheduleStage {
                         &mut execute_clock,
                         &mut provisioning_tracker_count,
                     );
-                    if let Some(ee) = maybe_ee {
+                    let should_break = if let Some(ee) = maybe_ee {
                         executing_queue_count = executing_queue_count.checked_add(1).unwrap();
                         to_execute_substage.send(ExecutablePayload(ee)).unwrap();
+
+                        false
+                    } else {
+                        true
                     }
 
                     if first_iteration {
@@ -1486,7 +1490,7 @@ impl ScheduleStage {
                     }
                     (empty_from, empty_from_exec) = (from_len == 0, from_exec_len == 0);
 
-                    if maybe_ee.is_none() {
+                    if should_break {
                         break;
                     }
                 }

From 97be11a53fb9a760914508c35127e580bcc1a511 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Wed, 7 Sep 2022 21:34:48 +0900
Subject: [PATCH 2266/3199] save

---
 scheduler/src/lib.rs | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 6bc82aa27ec58b..8bd5fb0c268c8d 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1471,11 +1471,10 @@ impl ScheduleStage {
                     let should_break = if let Some(ee) = maybe_ee {
                         executing_queue_count = executing_queue_count.checked_add(1).unwrap();
                         to_execute_substage.send(ExecutablePayload(ee)).unwrap();
-
                         false
                     } else {
                         true
-                    }
+                    };
 
                     if first_iteration {
                         first_iteration = false;

From 8e6e97b8154d76be0c71771eb7a38c3c7b83c959 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Wed, 7 Sep 2022 21:55:12 +0900
Subject: [PATCH 2267/3199] save

---
 scheduler/src/lib.rs | 36 +++++++++++++++---------------------
 1 file changed, 15 insertions(+), 21 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 8bd5fb0c268c8d..55a0c317d6f767 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1449,14 +1449,14 @@ impl ScheduleStage {
 
             loop {
                 let executing_like_count = executing_queue_count + provisioning_tracker_count;
-                while executing_like_count < max_executing_queue_count && empty_from_exec {
+                while executing_like_count < max_executing_queue_count || !address_book.uncontended_task_ids.is_empty() {
                     trace!("schedule_once id_{:016x} (prev: {}, exec: ({}|{}), runnnable: {}, contended: {}, (immediate+provisional)/max: ({}+{})/{}) active from contended: {} stuck: {} completed: {}!", random_id, from_prev.len(), to_execute_substage.len(), from_exec.len(), runnable_queue.task_count(), contended_count, executing_queue_count, provisioning_tracker_count, max_executing_queue_count, address_book.uncontended_task_ids.len(), address_book.stuck_tasks.len(), completed_count);
                     if start.elapsed() > std::time::Duration::from_millis(150) {
                         start = std::time::Instant::now();
                         info!("schedule_once:interval id_{:016x} (prev: {}, exec: ({}|{}), runnnable: {}, contended: {}, (immediate+provisional)/max: ({}+{})/{}) active from contended: {} stuck: {} completed: {}!", random_id, from_prev.len(), to_execute_substage.len(), from_exec.len(), runnable_queue.task_count(), contended_count, executing_queue_count, provisioning_tracker_count, max_executing_queue_count, address_book.uncontended_task_ids.len(), address_book.stuck_tasks.len(), completed_count);
                     }
                     let prefer_immediate = provisioning_tracker_count / 4 > executing_queue_count;
-                    let maybe_ee = Self::schedule_next_execution(
+                    if let Some(ee) = Self::schedule_next_execution(
                         ast,
                         &task_sender,
                         runnable_queue,
@@ -1467,32 +1467,26 @@ impl ScheduleStage {
                         &mut queue_clock,
                         &mut execute_clock,
                         &mut provisioning_tracker_count,
-                    );
-                    let should_break = if let Some(ee) = maybe_ee {
+                    ) {
                         executing_queue_count = executing_queue_count.checked_add(1).unwrap();
                         to_execute_substage.send(ExecutablePayload(ee)).unwrap();
-                        false
-                    } else {
-                        true
-                    };
-
-                    if first_iteration {
-                        first_iteration = false;
-                        (from_len, from_exec_len) = (from_prev.len(), from_exec.len());
                     } else {
-                        if empty_from {
-                            from_len = from_prev.len();
-                        }
-                        if empty_from_exec {
-                            from_exec_len = from_exec.len();
-                        }
+                        break;
                     }
-                    (empty_from, empty_from_exec) = (from_len == 0, from_exec_len == 0);
+                }
 
-                    if should_break {
-                        break;
+                if first_iteration {
+                    first_iteration = false;
+                    (from_len, from_exec_len) = (from_prev.len(), from_exec.len());
+                } else {
+                    if empty_from {
+                        from_len = from_prev.len();
+                    }
+                    if empty_from_exec {
+                        from_exec_len = from_exec.len();
                     }
                 }
+                (empty_from, empty_from_exec) = (from_len == 0, from_exec_len == 0);
 
                 if empty_from && empty_from_exec {
                     break;

From f6162b5a41cade3422e52a2b48d4f4beb95a0b6b Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Wed, 7 Sep 2022 22:03:31 +0900
Subject: [PATCH 2268/3199] save

---
 scheduler/src/lib.rs | 17 +++++++++++------
 1 file changed, 11 insertions(+), 6 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 55a0c317d6f767..afe8b0335c3544 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1332,15 +1332,16 @@ impl ScheduleStage {
         Self::push_to_runnable_queue(weighted_tx, runnable_queue)
     }
 
-    fn _run<AST: AtScheduleThread>(
+    fn _run<'a, AST: AtScheduleThread>(
         ast: AST,
         max_executing_queue_count: usize,
         runnable_queue: &mut TaskQueue,
         address_book: &mut AddressBook,
-        from_prev: &crossbeam_channel::Receiver<SchedulablePayload>,
+        mut from_prev: &'a, crossbeam_channel::Receiver<SchedulablePayload>,
         to_execute_substage: &crossbeam_channel::Sender<ExecutablePayload>,
         from_exec: &crossbeam_channel::Receiver<UnlockablePayload>,
         maybe_to_next_stage: Option<&crossbeam_channel::Sender<DroppablePayload>>, // assume nonblocking
+        never: &'a, crossbeam_channel::Receiver<SchedulablePayload>,
     ) {
         let random_id = rand::thread_rng().gen::<u64>();
         info!("schedule_once:initial id_{:016x}", random_id);
@@ -1412,6 +1413,7 @@ impl ScheduleStage {
 
         let (mut from_disconnected, mut from_exec_disconnected, mut no_more_work) = Default::default();
         loop {
+            if !from_disconnected  {
             crossbeam_channel::select! {
                recv(from_exec) -> maybe_from_exec => {
                    if let Ok(UnlockablePayload(mut processed_execution_environment)) = maybe_from_exec {
@@ -1434,15 +1436,17 @@ impl ScheduleStage {
                    } else {
                        assert_eq!(from_prev.len(), 0);
                        from_disconnected |= true;
-                       no_more_work |= runnable_queue.task_count() + contended_count + executing_queue_count + provisioning_tracker_count == 0;
+                       from_prev = never;
                        info!("flushing2..: {} {} {} {} {} {}", from_disconnected, from_exec_disconnected, runnable_queue.task_count(), contended_count,  executing_queue_count, provisioning_tracker_count);
-                       if from_exec_disconnected || no_more_work {
-                           break;
-                       }
                    }
                }
             }
 
+           no_more_work = from_disconnected && runnable_queue.task_count() + contended_count + executing_queue_count + provisioning_tracker_count == 0;
+           if from_disconnected && (from_exec_disconnected || no_more_work) {
+               break;
+           }
+
             let mut first_iteration = true;
             let (mut empty_from, mut empty_from_exec) = (false, false);
             let (mut from_len, mut from_exec_len) = (0, 0);
@@ -1552,6 +1556,7 @@ impl ScheduleStage {
             to_execute_substage,
             from_execute_substage,
             maybe_to_next_stage,
+            &crossbeam_channel::never(),
         )
     }
 }

From 07afee6dbc2eccc152a52607a38b7a031919a7f3 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Wed, 7 Sep 2022 22:03:58 +0900
Subject: [PATCH 2269/3199] save

---
 scheduler/src/lib.rs | 1 -
 1 file changed, 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index afe8b0335c3544..661d95b76df9ca 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1413,7 +1413,6 @@ impl ScheduleStage {
 
         let (mut from_disconnected, mut from_exec_disconnected, mut no_more_work) = Default::default();
         loop {
-            if !from_disconnected  {
             crossbeam_channel::select! {
                recv(from_exec) -> maybe_from_exec => {
                    if let Ok(UnlockablePayload(mut processed_execution_environment)) = maybe_from_exec {

From 3e1e25b9f89645c8d9dfed284f4ee2961e669269 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Wed, 7 Sep 2022 22:04:17 +0900
Subject: [PATCH 2270/3199] save

---
 scheduler/src/lib.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 661d95b76df9ca..f6bd6e1e984f5a 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1337,11 +1337,11 @@ impl ScheduleStage {
         max_executing_queue_count: usize,
         runnable_queue: &mut TaskQueue,
         address_book: &mut AddressBook,
-        mut from_prev: &'a, crossbeam_channel::Receiver<SchedulablePayload>,
+        mut from_prev: &'a crossbeam_channel::Receiver<SchedulablePayload>,
         to_execute_substage: &crossbeam_channel::Sender<ExecutablePayload>,
         from_exec: &crossbeam_channel::Receiver<UnlockablePayload>,
         maybe_to_next_stage: Option<&crossbeam_channel::Sender<DroppablePayload>>, // assume nonblocking
-        never: &'a, crossbeam_channel::Receiver<SchedulablePayload>,
+        never: &'a crossbeam_channel::Receiver<SchedulablePayload>,
     ) {
         let random_id = rand::thread_rng().gen::<u64>();
         info!("schedule_once:initial id_{:016x}", random_id);

From fc957e55ed2e7e3d9409d721734bae609e58fcaf Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Wed, 7 Sep 2022 22:12:10 +0900
Subject: [PATCH 2271/3199] save

---
 scheduler/src/lib.rs | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index f6bd6e1e984f5a..aad8aa63574a94 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1413,6 +1413,7 @@ impl ScheduleStage {
 
         let (mut from_disconnected, mut from_exec_disconnected, mut no_more_work) = Default::default();
         loop {
+            if !from_disconnected || executing_queue_count >= 1 {
             crossbeam_channel::select! {
                recv(from_exec) -> maybe_from_exec => {
                    if let Ok(UnlockablePayload(mut processed_execution_environment)) = maybe_from_exec {
@@ -1440,6 +1441,7 @@ impl ScheduleStage {
                    }
                }
             }
+            }
 
            no_more_work = from_disconnected && runnable_queue.task_count() + contended_count + executing_queue_count + provisioning_tracker_count == 0;
            if from_disconnected && (from_exec_disconnected || no_more_work) {

From 4917097d5c8dbed24b89b7d1359167f3944d7433 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Wed, 7 Sep 2022 22:13:45 +0900
Subject: [PATCH 2272/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index aad8aa63574a94..9b70c63039cea0 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1411,7 +1411,7 @@ impl ScheduleStage {
         }).collect::<Vec<_>>();
         let mut start = std::time::Instant::now();
 
-        let (mut from_disconnected, mut from_exec_disconnected, mut no_more_work) = Default::default();
+        let (mut from_disconnected, mut from_exec_disconnected, mut no_more_work): (bool, bool, bool) = Default::default();
         loop {
             if !from_disconnected || executing_queue_count >= 1 {
             crossbeam_channel::select! {

From e7da94fe2b7789495bc7fa1e1b5acbc2b2470246 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 8 Sep 2022 11:07:29 +0900
Subject: [PATCH 2273/3199] save

---
 scheduler/src/lib.rs | 15 ++++++++++-----
 1 file changed, 10 insertions(+), 5 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 9b70c63039cea0..c752012ff1f08d 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -889,9 +889,13 @@ impl ScheduleStage {
                 let uw = weight_from_contended.key();
 
                 if weight_from_runnable > uw {
+                    panic!("replay shouldn't see this branch");
+
+                    /*
                     trace!("select: runnable > contended");
                     let t = heaviest_runnable_entry.remove();
                     Some((TaskSource::Runnable, t))
+                    */
                 } else if uw > weight_from_runnable {
                     trace!("select: contended > runnnable");
                     let t = weight_from_contended.remove();
@@ -1014,6 +1018,7 @@ impl ScheduleStage {
                                 .load(std::sync::atomic::Ordering::SeqCst)
                         );
                         //address_book.uncontended_task_ids.clear();
+                        address_book.uncontended_task_ids.insert(next_task.unique_weight, next_task);
                     }
 
                     if from_runnable || task_source == TaskSource::Stuck {
@@ -1054,6 +1059,7 @@ impl ScheduleStage {
                             .insert(next_task.stuck_task_id(), removed);
                         assert!(a.is_none());
                         */
+                        
                         break;
                     }
                 } else if provisional_count > 0 {
@@ -1424,7 +1430,7 @@ impl ScheduleStage {
                    } else {
                        assert_eq!(from_exec.len(), 0);
                        from_exec_disconnected |= true;
-                       info!("flushing1..: {} {} {} {} {} {}", from_disconnected, from_exec_disconnected, runnable_queue.task_count(), contended_count,  executing_queue_count, provisioning_tracker_count);
+                       info!("flushing1..: {} {} {} {} {}", (from_disconnected, from_exec_disconnected), runnable_queue.task_count(), contended_count, executing_queue_count, provisioning_tracker_count);
                        if from_disconnected {
                            break;
                        }
@@ -1437,7 +1443,7 @@ impl ScheduleStage {
                        assert_eq!(from_prev.len(), 0);
                        from_disconnected |= true;
                        from_prev = never;
-                       info!("flushing2..: {} {} {} {} {} {}", from_disconnected, from_exec_disconnected, runnable_queue.task_count(), contended_count,  executing_queue_count, provisioning_tracker_count);
+                       info!("flushing2..: {} {} {} {} {}", (from_disconnected, from_exec_disconnected), runnable_queue.task_count(), contended_count, executing_queue_count, provisioning_tracker_count);
                    }
                }
             }
@@ -1454,13 +1460,14 @@ impl ScheduleStage {
 
             loop {
                 let executing_like_count = executing_queue_count + provisioning_tracker_count;
-                while executing_like_count < max_executing_queue_count || !address_book.uncontended_task_ids.is_empty() {
+                if executing_like_count < max_executing_queue_count {
                     trace!("schedule_once id_{:016x} (prev: {}, exec: ({}|{}), runnnable: {}, contended: {}, (immediate+provisional)/max: ({}+{})/{}) active from contended: {} stuck: {} completed: {}!", random_id, from_prev.len(), to_execute_substage.len(), from_exec.len(), runnable_queue.task_count(), contended_count, executing_queue_count, provisioning_tracker_count, max_executing_queue_count, address_book.uncontended_task_ids.len(), address_book.stuck_tasks.len(), completed_count);
                     if start.elapsed() > std::time::Duration::from_millis(150) {
                         start = std::time::Instant::now();
                         info!("schedule_once:interval id_{:016x} (prev: {}, exec: ({}|{}), runnnable: {}, contended: {}, (immediate+provisional)/max: ({}+{})/{}) active from contended: {} stuck: {} completed: {}!", random_id, from_prev.len(), to_execute_substage.len(), from_exec.len(), runnable_queue.task_count(), contended_count, executing_queue_count, provisioning_tracker_count, max_executing_queue_count, address_book.uncontended_task_ids.len(), address_book.stuck_tasks.len(), completed_count);
                     }
                     let prefer_immediate = provisioning_tracker_count / 4 > executing_queue_count;
+
                     if let Some(ee) = Self::schedule_next_execution(
                         ast,
                         &task_sender,
@@ -1475,8 +1482,6 @@ impl ScheduleStage {
                     ) {
                         executing_queue_count = executing_queue_count.checked_add(1).unwrap();
                         to_execute_substage.send(ExecutablePayload(ee)).unwrap();
-                    } else {
-                        break;
                     }
                 }
 

From 674536fc918cf85810506114dbda378957e73bf9 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 8 Sep 2022 11:07:50 +0900
Subject: [PATCH 2274/3199] save

---
 scheduler/src/lib.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index c752012ff1f08d..d232114b3af194 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1430,7 +1430,7 @@ impl ScheduleStage {
                    } else {
                        assert_eq!(from_exec.len(), 0);
                        from_exec_disconnected |= true;
-                       info!("flushing1..: {} {} {} {} {}", (from_disconnected, from_exec_disconnected), runnable_queue.task_count(), contended_count, executing_queue_count, provisioning_tracker_count);
+                       info!("flushing1..: {:?} {} {} {} {}", (from_disconnected, from_exec_disconnected), runnable_queue.task_count(), contended_count, executing_queue_count, provisioning_tracker_count);
                        if from_disconnected {
                            break;
                        }
@@ -1443,7 +1443,7 @@ impl ScheduleStage {
                        assert_eq!(from_prev.len(), 0);
                        from_disconnected |= true;
                        from_prev = never;
-                       info!("flushing2..: {} {} {} {} {}", (from_disconnected, from_exec_disconnected), runnable_queue.task_count(), contended_count, executing_queue_count, provisioning_tracker_count);
+                       info!("flushing2..: {:?} {} {} {} {}", (from_disconnected, from_exec_disconnected), runnable_queue.task_count(), contended_count, executing_queue_count, provisioning_tracker_count);
                    }
                }
             }

From 2f7b932e5806ff63b5423fc85b67bc58a9158a5d Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 8 Sep 2022 11:08:38 +0900
Subject: [PATCH 2275/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index d232114b3af194..71cc80c4c6bd26 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1018,7 +1018,6 @@ impl ScheduleStage {
                                 .load(std::sync::atomic::Ordering::SeqCst)
                         );
                         //address_book.uncontended_task_ids.clear();
-                        address_book.uncontended_task_ids.insert(next_task.unique_weight, next_task);
                     }
 
                     if from_runnable || task_source == TaskSource::Stuck {
@@ -1059,6 +1058,7 @@ impl ScheduleStage {
                             .insert(next_task.stuck_task_id(), removed);
                         assert!(a.is_none());
                         */
+                        address_book.uncontended_task_ids.insert(next_task.unique_weight, next_task);
                         
                         break;
                     }

From 5e551eef1603d1e53ddea7f69a8a377f86a5c539 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 8 Sep 2022 11:12:40 +0900
Subject: [PATCH 2276/3199] save

---
 scheduler/src/lib.rs | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 71cc80c4c6bd26..68c7030d4e38ba 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1045,7 +1045,7 @@ impl ScheduleStage {
                         } else {
                             unreachable!();
                         }
-                    } else {
+                    } else if task_source == TaskSource::Contended {
                         // todo: remove this task from stuck_tasks before update_busiest_page_cu
                         /*
                         let removed = address_book
@@ -1061,6 +1061,8 @@ impl ScheduleStage {
                         address_book.uncontended_task_ids.insert(next_task.unique_weight, next_task);
                         
                         break;
+                    } else {
+                        unreachable!();
                     }
                 } else if provisional_count > 0 {
                     assert!(!from_runnable);

From f4beca505cc6cd7124f491a3d7467a8c40df10b5 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 8 Sep 2022 11:30:42 +0900
Subject: [PATCH 2277/3199] save

---
 runtime/src/bank.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index c1474ecf71244d..38c1bef3e252e2 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1316,7 +1316,7 @@ impl Default for Scheduler {
                         ("thread", current_thread_name, String),
                         ("signature", &sig, String),
                         ("account_locks_in_json", "{}", String),
-                        ("status", "Ok", String),
+                        ("status", format!("{}", ee.execution_result), String),
                         ("duration", duration_with_overhead, i64),
                         ("compute_units", ee.cu, i64),
                     );

From 99017d448f1e8c66f5803d58ebc3c457757b8c44 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 8 Sep 2022 11:32:07 +0900
Subject: [PATCH 2278/3199] save

---
 runtime/src/bank.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 38c1bef3e252e2..6e47e75c0f6dd0 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1316,7 +1316,7 @@ impl Default for Scheduler {
                         ("thread", current_thread_name, String),
                         ("signature", &sig, String),
                         ("account_locks_in_json", "{}", String),
-                        ("status", format!("{}", ee.execution_result), String),
+                        ("status", format!("{:?}", ee.execution_result), String),
                         ("duration", duration_with_overhead, i64),
                         ("compute_units", ee.cu, i64),
                     );

From 2ca7b922dccde6636734641aa3215a50ae419f69 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 8 Sep 2022 11:40:53 +0900
Subject: [PATCH 2279/3199] save

---
 runtime/src/bank.rs | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 6e47e75c0f6dd0..a44cd5d321c30d 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1297,10 +1297,12 @@ impl Default for Scheduler {
 
                 let TransactionResults {
                     fee_collection_results,
+                    execution_results,
                     ..
                 } = tx_results;
 
                 ee.execution_result = Some(fee_collection_results.into_iter().collect::<Result<_>>());
+                ee.cu = execution_results[0].details().unwrap().executed_units;
 
 
                 //ee.reindex_with_address_book();

From 65dc91c7f8f14f985a218e84c27c5d3d18747e28 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 8 Sep 2022 12:27:10 +0900
Subject: [PATCH 2280/3199] save

---
 scheduler/src/lib.rs | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 68c7030d4e38ba..b8a2f8e92fa7ec 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1463,11 +1463,6 @@ impl ScheduleStage {
             loop {
                 let executing_like_count = executing_queue_count + provisioning_tracker_count;
                 if executing_like_count < max_executing_queue_count {
-                    trace!("schedule_once id_{:016x} (prev: {}, exec: ({}|{}), runnnable: {}, contended: {}, (immediate+provisional)/max: ({}+{})/{}) active from contended: {} stuck: {} completed: {}!", random_id, from_prev.len(), to_execute_substage.len(), from_exec.len(), runnable_queue.task_count(), contended_count, executing_queue_count, provisioning_tracker_count, max_executing_queue_count, address_book.uncontended_task_ids.len(), address_book.stuck_tasks.len(), completed_count);
-                    if start.elapsed() > std::time::Duration::from_millis(150) {
-                        start = std::time::Instant::now();
-                        info!("schedule_once:interval id_{:016x} (prev: {}, exec: ({}|{}), runnnable: {}, contended: {}, (immediate+provisional)/max: ({}+{})/{}) active from contended: {} stuck: {} completed: {}!", random_id, from_prev.len(), to_execute_substage.len(), from_exec.len(), runnable_queue.task_count(), contended_count, executing_queue_count, provisioning_tracker_count, max_executing_queue_count, address_book.uncontended_task_ids.len(), address_book.stuck_tasks.len(), completed_count);
-                    }
                     let prefer_immediate = provisioning_tracker_count / 4 > executing_queue_count;
 
                     if let Some(ee) = Self::schedule_next_execution(
@@ -1485,6 +1480,11 @@ impl ScheduleStage {
                         executing_queue_count = executing_queue_count.checked_add(1).unwrap();
                         to_execute_substage.send(ExecutablePayload(ee)).unwrap();
                     }
+                    trace!("schedule_once id_{:016x} ch(prev: {}, exec: {}|{}), runnnable: {}, contended: {}, (immediate+provisional)/max: ({}+{})/{} uncontended: {} stuck: {} completed: {}!", random_id, from_prev.len(), to_execute_substage.len(), from_exec.len(), runnable_queue.task_count(), contended_count, executing_queue_count, provisioning_tracker_count, max_executing_queue_count, address_book.uncontended_task_ids.len(), address_book.stuck_tasks.len(), completed_count);
+                    if start.elapsed() > std::time::Duration::from_millis(150) {
+                        start = std::time::Instant::now();
+                        info!("schedule_once:interval id_{:016x} ch(prev: {}, exec: {}|{}), runnnable: {}, contended: {}, (immediate+provisional)/max: ({}+{})/{} uncontended: {} stuck: {} completed: {}!", random_id, from_prev.len(), to_execute_substage.len(), from_exec.len(), runnable_queue.task_count(), contended_count, executing_queue_count, provisioning_tracker_count, max_executing_queue_count, address_book.uncontended_task_ids.len(), address_book.stuck_tasks.len(), completed_count);
+                    }
                 }
 
                 if first_iteration {
@@ -1539,7 +1539,7 @@ impl ScheduleStage {
             indexer_handle.join().unwrap().unwrap();
         }
 
-        info!("schedule_once:final id_{:016x} (from_disconnected: {}, from_exec_disconnected: {}, no_more_work: {}) (prev: {}, exec: ({}|{}), runnnable: {}, contended: {}, (immediate+provisional)/max: ({}+{})/{}) active from contended: {} stuck: {} completed: {}!", random_id, from_disconnected, from_exec_disconnected, no_more_work, from_prev.len(), to_execute_substage.len(), from_exec.len(), runnable_queue.task_count(), contended_count, executing_queue_count, provisioning_tracker_count, max_executing_queue_count, address_book.uncontended_task_ids.len(), address_book.stuck_tasks.len(), completed_count);
+        info!("schedule_once:final id_{:016x} (from_disconnected: {}, from_exec_disconnected: {}, no_more_work: {}) ch(prev: {}, exec: {}|{}), runnnable: {}, contended: {}, (immediate+provisional)/max: ({}+{})/{} uncontended: {} stuck: {} completed: {}!", random_id, from_disconnected, from_exec_disconnected, no_more_work, from_prev.len(), to_execute_substage.len(), from_exec.len(), runnable_queue.task_count(), contended_count, executing_queue_count, provisioning_tracker_count, max_executing_queue_count, address_book.uncontended_task_ids.len(), address_book.stuck_tasks.len(), completed_count);
     }
 
     pub fn run(

From 9432bb4680714cccae3d4a40b8ead1992d4941bf Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 8 Sep 2022 13:17:28 +0900
Subject: [PATCH 2281/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index b8a2f8e92fa7ec..b40074cf39b04f 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -889,7 +889,7 @@ impl ScheduleStage {
                 let uw = weight_from_contended.key();
 
                 if weight_from_runnable > uw {
-                    panic!("replay shouldn't see this branch");
+                    panic!("replay shouldn't see this branch: {} > {}", weight_from_runnable, uw);
 
                     /*
                     trace!("select: runnable > contended");

From f164ce20becd27654b53fd5a38fe1864d49ec968 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 8 Sep 2022 13:31:57 +0900
Subject: [PATCH 2282/3199] save

---
 runtime/src/bank.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index a44cd5d321c30d..3a716213418337 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1318,7 +1318,7 @@ impl Default for Scheduler {
                         ("thread", current_thread_name, String),
                         ("signature", &sig, String),
                         ("account_locks_in_json", "{}", String),
-                        ("status", format!("{:?}", ee.execution_result), String),
+                        ("status", format!("{:?}", ee.execution_result.unwrap()), String),
                         ("duration", duration_with_overhead, i64),
                         ("compute_units", ee.cu, i64),
                     );

From 27b2a4d9d8ba1da40b65cbbae2d6bd75e5ddbdc6 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 8 Sep 2022 13:35:58 +0900
Subject: [PATCH 2283/3199] save

---
 runtime/src/bank.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 3a716213418337..707fecac81d7f5 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1318,7 +1318,7 @@ impl Default for Scheduler {
                         ("thread", current_thread_name, String),
                         ("signature", &sig, String),
                         ("account_locks_in_json", "{}", String),
-                        ("status", format!("{:?}", ee.execution_result.unwrap()), String),
+                        ("status", format!("{:?}", ee.execution_result.as_ref().unwrap()), String),
                         ("duration", duration_with_overhead, i64),
                         ("compute_units", ee.cu, i64),
                     );

From 726996bf3e542e6823f8132b4164ac033b8cec44 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 8 Sep 2022 13:45:29 +0900
Subject: [PATCH 2284/3199] save

---
 ledger/src/blockstore_processor.rs | 3 ++-
 scheduler/src/lib.rs               | 1 +
 2 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/ledger/src/blockstore_processor.rs b/ledger/src/blockstore_processor.rs
index d2799b8019d9cf..13e36fd2260708 100644
--- a/ledger/src/blockstore_processor.rs
+++ b/ledger/src/blockstore_processor.rs
@@ -1235,6 +1235,7 @@ fn confirm_slot_entries(
         Arc::new(verify_transaction),
     );
     let transaction_cpu_duration_us = timing::duration_as_us(&check_start.elapsed());
+    let disable_randomize = std::env::var("DISABLE_RANDOMIZE").is_ok();
 
     match check_result {
         Ok(mut check_result) => {
@@ -1256,7 +1257,7 @@ fn confirm_slot_entries(
             let process_result = process_entries_with_callback(
                 bank,
                 &mut replay_entries,
-                true, // shuffle transactions.
+                !disable_randomize, // shuffle transactions.
                 entry_callback,
                 transaction_status_sender,
                 replay_vote_sender,
diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index b40074cf39b04f..8b4fe8c81e200b 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1336,6 +1336,7 @@ impl ScheduleStage {
         sequence_time: &mut usize,
     ) {
         weighted_tx.record_sequence_time(*sequence_time);
+        assert!(sequence_time, weighted_tx.transaction_index_in_entries_for_replay())
         *sequence_time = sequence_time.checked_add(1).unwrap();
         Self::push_to_runnable_queue(weighted_tx, runnable_queue)
     }

From 1928b6d8548e8361a96d5050c7ef80886e5e28a4 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 8 Sep 2022 13:45:44 +0900
Subject: [PATCH 2285/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 8b4fe8c81e200b..6ad94f66730096 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1336,7 +1336,7 @@ impl ScheduleStage {
         sequence_time: &mut usize,
     ) {
         weighted_tx.record_sequence_time(*sequence_time);
-        assert!(sequence_time, weighted_tx.transaction_index_in_entries_for_replay())
+        assert!(*sequence_time, weighted_tx.transaction_index_in_entries_for_replay())
         *sequence_time = sequence_time.checked_add(1).unwrap();
         Self::push_to_runnable_queue(weighted_tx, runnable_queue)
     }

From c0ab830d3c4b4ec68892460e4fa31a17de024cd6 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 8 Sep 2022 13:45:53 +0900
Subject: [PATCH 2286/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 6ad94f66730096..e6d6052679b215 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1336,7 +1336,7 @@ impl ScheduleStage {
         sequence_time: &mut usize,
     ) {
         weighted_tx.record_sequence_time(*sequence_time);
-        assert!(*sequence_time, weighted_tx.transaction_index_in_entries_for_replay())
+        assert_eq!(*sequence_time, weighted_tx.transaction_index_in_entries_for_replay())
         *sequence_time = sequence_time.checked_add(1).unwrap();
         Self::push_to_runnable_queue(weighted_tx, runnable_queue)
     }

From f59c0948d430a664a4a75f778d462c7e1c6647ed Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 8 Sep 2022 13:46:06 +0900
Subject: [PATCH 2287/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index e6d6052679b215..15e7bbde235837 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1336,7 +1336,7 @@ impl ScheduleStage {
         sequence_time: &mut usize,
     ) {
         weighted_tx.record_sequence_time(*sequence_time);
-        assert_eq!(*sequence_time, weighted_tx.transaction_index_in_entries_for_replay())
+        assert_eq!(*sequence_time, weighted_tx.transaction_index_in_entries_for_replay() as usize)
         *sequence_time = sequence_time.checked_add(1).unwrap();
         Self::push_to_runnable_queue(weighted_tx, runnable_queue)
     }

From b86e3ce723ec281ff03f662abbc91b7089763fcf Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 8 Sep 2022 13:46:19 +0900
Subject: [PATCH 2288/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 15e7bbde235837..ca272296c77523 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1336,7 +1336,7 @@ impl ScheduleStage {
         sequence_time: &mut usize,
     ) {
         weighted_tx.record_sequence_time(*sequence_time);
-        assert_eq!(*sequence_time, weighted_tx.transaction_index_in_entries_for_replay() as usize)
+        assert_eq!(*sequence_time, (weighted_tx.transaction_index_in_entries_for_replay() as usize))
         *sequence_time = sequence_time.checked_add(1).unwrap();
         Self::push_to_runnable_queue(weighted_tx, runnable_queue)
     }

From 683f11f5c4f2553083a63761238402d6e6bc5318 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 8 Sep 2022 13:46:39 +0900
Subject: [PATCH 2289/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index ca272296c77523..317c20f31d8bd8 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1336,7 +1336,7 @@ impl ScheduleStage {
         sequence_time: &mut usize,
     ) {
         weighted_tx.record_sequence_time(*sequence_time);
-        assert_eq!(*sequence_time, (weighted_tx.transaction_index_in_entries_for_replay() as usize))
+        assert_eq!(*sequence_time, weighted_tx.transaction_index_in_entries_for_replay() as usize);
         *sequence_time = sequence_time.checked_add(1).unwrap();
         Self::push_to_runnable_queue(weighted_tx, runnable_queue)
     }

From 6be032172e24562707a4d940b8c52ad501069797 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 8 Sep 2022 14:28:22 +0900
Subject: [PATCH 2290/3199] save

---
 scheduler/src/lib.rs | 54 ++++++++++++++++++++++++++++++++++++++++----
 1 file changed, 49 insertions(+), 5 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 317c20f31d8bd8..f1207f32d203ef 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -869,6 +869,7 @@ impl ScheduleStage {
         runnable_queue: &'a mut TaskQueue,
         address_book: &mut AddressBook,
         contended_count: &usize,
+        runnable_exclusive: bool,
     ) -> Option<(TaskSource, TaskInQueue)> {
         match (
             runnable_queue.heaviest_entry_to_execute(),
@@ -877,12 +878,20 @@ impl ScheduleStage {
             (Some(heaviest_runnable_entry), None) => {
                 trace!("select: runnable only");
                 let t = heaviest_runnable_entry.remove();
-                Some((TaskSource::Runnable, t))
+                if runnable_exclusive {
+                    Some((TaskSource::Runnable, t))
+                } else {
+                    None
+                }
             }
             (None, Some(weight_from_contended)) => {
                 trace!("select: contended only");
                 let t = weight_from_contended.remove();
-                Some((TaskSource::Contended, t))
+                if runnable_exclusive {
+                    None
+                } else {
+                    Some((TaskSource::Contended, t))
+                }
             }
             (Some(heaviest_runnable_entry), Some(weight_from_contended)) => {
                 let weight_from_runnable = heaviest_runnable_entry.key();
@@ -899,7 +908,11 @@ impl ScheduleStage {
                 } else if uw > weight_from_runnable {
                     trace!("select: contended > runnnable");
                     let t = weight_from_contended.remove();
-                    Some((TaskSource::Contended, t))
+                    if runnable_exclusive {
+                        None
+                    } else {
+                        Some((TaskSource::Contended, t))
+                    }
                 } else {
                     unreachable!(
                         "identical unique weights shouldn't exist in both runnable and contended"
@@ -908,6 +921,7 @@ impl ScheduleStage {
             }
             (None, None) => {
                 trace!("select: none");
+
                 if false && runnable_queue.task_count() == 0 && /* *contended_count > 0 &&*/ address_book.stuck_tasks.len() > 0
                 {
                     trace!("handling stuck...");
@@ -940,6 +954,7 @@ impl ScheduleStage {
         sequence_clock: &usize,
         queue_clock: &mut usize,
         provisioning_tracker_count: &mut usize,
+        runnable_exclusive: bool,
     ) -> Option<(UniqueWeight, TaskInQueue, Vec<LockAttempt>)> {
         if let Some(mut a) = address_book.fulfilled_provisional_task_ids.pop_last() {
             trace!(
@@ -955,7 +970,7 @@ impl ScheduleStage {
         trace!("pop begin");
         loop {
             if let Some((task_source, mut next_task)) =
-                Self::select_next_task(runnable_queue, address_book, contended_count)
+                Self::select_next_task(runnable_queue, address_book, contended_count, runnable_exclusive)
             {
                 trace!("pop loop iteration");
                 let from_runnable = task_source == TaskSource::Runnable;
@@ -1311,6 +1326,7 @@ impl ScheduleStage {
         queue_clock: &mut usize,
         execute_clock: &mut usize,
         provisioning_tracker_count: &mut usize,
+        runnable_exclusive: bool,
     ) -> Option<Box<ExecutionEnvironment>> {
         let maybe_ee = Self::pop_from_queue_then_lock(
             ast,
@@ -1322,6 +1338,7 @@ impl ScheduleStage {
             sequence_time,
             queue_clock,
             provisioning_tracker_count,
+            runnable_exclusive,
         )
         .map(|(uw, t, ll)| {
             Self::prepare_scheduled_execution(address_book, uw, t, ll, queue_clock, execute_clock)
@@ -1477,11 +1494,38 @@ impl ScheduleStage {
                         &mut queue_clock,
                         &mut execute_clock,
                         &mut provisioning_tracker_count,
+                        true
+                    ) {
+                        executing_queue_count = executing_queue_count.checked_add(1).unwrap();
+                        to_execute_substage.send(ExecutablePayload(ee)).unwrap();
+                    }
+                    trace!("schedule_once runnable id_{:016x} ch(prev: {}, exec: {}|{}), runnnable: {}, contended: {}, (immediate+provisional)/max: ({}+{})/{} uncontended: {} stuck: {} completed: {}!", random_id, from_prev.len(), to_execute_substage.len(), from_exec.len(), runnable_queue.task_count(), contended_count, executing_queue_count, provisioning_tracker_count, max_executing_queue_count, address_book.uncontended_task_ids.len(), address_book.stuck_tasks.len(), completed_count);
+                    if start.elapsed() > std::time::Duration::from_millis(150) {
+                        start = std::time::Instant::now();
+                        info!("schedule_once:interval id_{:016x} ch(prev: {}, exec: {}|{}), runnnable: {}, contended: {}, (immediate+provisional)/max: ({}+{})/{} uncontended: {} stuck: {} completed: {}!", random_id, from_prev.len(), to_execute_substage.len(), from_exec.len(), runnable_queue.task_count(), contended_count, executing_queue_count, provisioning_tracker_count, max_executing_queue_count, address_book.uncontended_task_ids.len(), address_book.stuck_tasks.len(), completed_count);
+                    }
+                }
+                let executing_like_count = executing_queue_count + provisioning_tracker_count;
+                if executing_like_count < max_executing_queue_count {
+                    let prefer_immediate = provisioning_tracker_count / 4 > executing_queue_count;
+
+                    if let Some(ee) = Self::schedule_next_execution(
+                        ast,
+                        &task_sender,
+                        runnable_queue,
+                        address_book,
+                        &mut contended_count,
+                        prefer_immediate,
+                        &sequence_time,
+                        &mut queue_clock,
+                        &mut execute_clock,
+                        &mut provisioning_tracker_count,
+                        false,
                     ) {
                         executing_queue_count = executing_queue_count.checked_add(1).unwrap();
                         to_execute_substage.send(ExecutablePayload(ee)).unwrap();
                     }
-                    trace!("schedule_once id_{:016x} ch(prev: {}, exec: {}|{}), runnnable: {}, contended: {}, (immediate+provisional)/max: ({}+{})/{} uncontended: {} stuck: {} completed: {}!", random_id, from_prev.len(), to_execute_substage.len(), from_exec.len(), runnable_queue.task_count(), contended_count, executing_queue_count, provisioning_tracker_count, max_executing_queue_count, address_book.uncontended_task_ids.len(), address_book.stuck_tasks.len(), completed_count);
+                    trace!("schedule_once contended id_{:016x} ch(prev: {}, exec: {}|{}), runnnable: {}, contended: {}, (immediate+provisional)/max: ({}+{})/{} uncontended: {} stuck: {} completed: {}!", random_id, from_prev.len(), to_execute_substage.len(), from_exec.len(), runnable_queue.task_count(), contended_count, executing_queue_count, provisioning_tracker_count, max_executing_queue_count, address_book.uncontended_task_ids.len(), address_book.stuck_tasks.len(), completed_count);
                     if start.elapsed() > std::time::Duration::from_millis(150) {
                         start = std::time::Instant::now();
                         info!("schedule_once:interval id_{:016x} ch(prev: {}, exec: {}|{}), runnnable: {}, contended: {}, (immediate+provisional)/max: ({}+{})/{} uncontended: {} stuck: {} completed: {}!", random_id, from_prev.len(), to_execute_substage.len(), from_exec.len(), runnable_queue.task_count(), contended_count, executing_queue_count, provisioning_tracker_count, max_executing_queue_count, address_book.uncontended_task_ids.len(), address_book.stuck_tasks.len(), completed_count);

From f40e69a7445b6aa1ed2dfcda1532883defbb71ee Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 8 Sep 2022 14:45:07 +0900
Subject: [PATCH 2291/3199] save

---
 scheduler/src/lib.rs | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index f1207f32d203ef..cc18d3ad15b68c 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -877,8 +877,8 @@ impl ScheduleStage {
         ) {
             (Some(heaviest_runnable_entry), None) => {
                 trace!("select: runnable only");
-                let t = heaviest_runnable_entry.remove();
                 if runnable_exclusive {
+                    let t = heaviest_runnable_entry.remove();
                     Some((TaskSource::Runnable, t))
                 } else {
                     None
@@ -886,10 +886,10 @@ impl ScheduleStage {
             }
             (None, Some(weight_from_contended)) => {
                 trace!("select: contended only");
-                let t = weight_from_contended.remove();
                 if runnable_exclusive {
                     None
                 } else {
+                    let t = weight_from_contended.remove();
                     Some((TaskSource::Contended, t))
                 }
             }
@@ -907,10 +907,10 @@ impl ScheduleStage {
                     */
                 } else if uw > weight_from_runnable {
                     trace!("select: contended > runnnable");
-                    let t = weight_from_contended.remove();
                     if runnable_exclusive {
                         None
                     } else {
+                        let t = weight_from_contended.remove();
                         Some((TaskSource::Contended, t))
                     }
                 } else {

From 3292d2d2a8ec705455d63520b27a63f80d7b8ea2 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 8 Sep 2022 14:46:05 +0900
Subject: [PATCH 2292/3199] save

---
 scheduler/src/lib.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index cc18d3ad15b68c..76d954d9f4987c 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1499,7 +1499,7 @@ impl ScheduleStage {
                         executing_queue_count = executing_queue_count.checked_add(1).unwrap();
                         to_execute_substage.send(ExecutablePayload(ee)).unwrap();
                     }
-                    trace!("schedule_once runnable id_{:016x} ch(prev: {}, exec: {}|{}), runnnable: {}, contended: {}, (immediate+provisional)/max: ({}+{})/{} uncontended: {} stuck: {} completed: {}!", random_id, from_prev.len(), to_execute_substage.len(), from_exec.len(), runnable_queue.task_count(), contended_count, executing_queue_count, provisioning_tracker_count, max_executing_queue_count, address_book.uncontended_task_ids.len(), address_book.stuck_tasks.len(), completed_count);
+                    trace!("schedule_once [R] id_{:016x} ch(prev: {}, exec: {}|{}), runnnable: {}, contended: {}, (immediate+provisional)/max: ({}+{})/{} uncontended: {} stuck: {} completed: {}!", random_id, from_prev.len(), to_execute_substage.len(), from_exec.len(), runnable_queue.task_count(), contended_count, executing_queue_count, provisioning_tracker_count, max_executing_queue_count, address_book.uncontended_task_ids.len(), address_book.stuck_tasks.len(), completed_count);
                     if start.elapsed() > std::time::Duration::from_millis(150) {
                         start = std::time::Instant::now();
                         info!("schedule_once:interval id_{:016x} ch(prev: {}, exec: {}|{}), runnnable: {}, contended: {}, (immediate+provisional)/max: ({}+{})/{} uncontended: {} stuck: {} completed: {}!", random_id, from_prev.len(), to_execute_substage.len(), from_exec.len(), runnable_queue.task_count(), contended_count, executing_queue_count, provisioning_tracker_count, max_executing_queue_count, address_book.uncontended_task_ids.len(), address_book.stuck_tasks.len(), completed_count);
@@ -1525,7 +1525,7 @@ impl ScheduleStage {
                         executing_queue_count = executing_queue_count.checked_add(1).unwrap();
                         to_execute_substage.send(ExecutablePayload(ee)).unwrap();
                     }
-                    trace!("schedule_once contended id_{:016x} ch(prev: {}, exec: {}|{}), runnnable: {}, contended: {}, (immediate+provisional)/max: ({}+{})/{} uncontended: {} stuck: {} completed: {}!", random_id, from_prev.len(), to_execute_substage.len(), from_exec.len(), runnable_queue.task_count(), contended_count, executing_queue_count, provisioning_tracker_count, max_executing_queue_count, address_book.uncontended_task_ids.len(), address_book.stuck_tasks.len(), completed_count);
+                    trace!("schedule_once [C] id_{:016x} ch(prev: {}, exec: {}|{}), runnnable: {}, contended: {}, (immediate+provisional)/max: ({}+{})/{} uncontended: {} stuck: {} completed: {}!", random_id, from_prev.len(), to_execute_substage.len(), from_exec.len(), runnable_queue.task_count(), contended_count, executing_queue_count, provisioning_tracker_count, max_executing_queue_count, address_book.uncontended_task_ids.len(), address_book.stuck_tasks.len(), completed_count);
                     if start.elapsed() > std::time::Duration::from_millis(150) {
                         start = std::time::Instant::now();
                         info!("schedule_once:interval id_{:016x} ch(prev: {}, exec: {}|{}), runnnable: {}, contended: {}, (immediate+provisional)/max: ({}+{})/{} uncontended: {} stuck: {} completed: {}!", random_id, from_prev.len(), to_execute_substage.len(), from_exec.len(), runnable_queue.task_count(), contended_count, executing_queue_count, provisioning_tracker_count, max_executing_queue_count, address_book.uncontended_task_ids.len(), address_book.stuck_tasks.len(), completed_count);

From 08eb5bde104d4407aa7e80e8a65d48cdad81c3ae Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 8 Sep 2022 14:46:39 +0900
Subject: [PATCH 2293/3199] save

---
 scheduler/src/lib.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 76d954d9f4987c..282fec558c531d 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1499,7 +1499,7 @@ impl ScheduleStage {
                         executing_queue_count = executing_queue_count.checked_add(1).unwrap();
                         to_execute_substage.send(ExecutablePayload(ee)).unwrap();
                     }
-                    trace!("schedule_once [R] id_{:016x} ch(prev: {}, exec: {}|{}), runnnable: {}, contended: {}, (immediate+provisional)/max: ({}+{})/{} uncontended: {} stuck: {} completed: {}!", random_id, from_prev.len(), to_execute_substage.len(), from_exec.len(), runnable_queue.task_count(), contended_count, executing_queue_count, provisioning_tracker_count, max_executing_queue_count, address_book.uncontended_task_ids.len(), address_book.stuck_tasks.len(), completed_count);
+                    trace!("schedule_once id_{:016x} [R] ch(prev: {}, exec: {}|{}), runnnable: {}, contended: {}, (immediate+provisional)/max: ({}+{})/{} uncontended: {} stuck: {} completed: {}!", random_id, from_prev.len(), to_execute_substage.len(), from_exec.len(), runnable_queue.task_count(), contended_count, executing_queue_count, provisioning_tracker_count, max_executing_queue_count, address_book.uncontended_task_ids.len(), address_book.stuck_tasks.len(), completed_count);
                     if start.elapsed() > std::time::Duration::from_millis(150) {
                         start = std::time::Instant::now();
                         info!("schedule_once:interval id_{:016x} ch(prev: {}, exec: {}|{}), runnnable: {}, contended: {}, (immediate+provisional)/max: ({}+{})/{} uncontended: {} stuck: {} completed: {}!", random_id, from_prev.len(), to_execute_substage.len(), from_exec.len(), runnable_queue.task_count(), contended_count, executing_queue_count, provisioning_tracker_count, max_executing_queue_count, address_book.uncontended_task_ids.len(), address_book.stuck_tasks.len(), completed_count);
@@ -1525,7 +1525,7 @@ impl ScheduleStage {
                         executing_queue_count = executing_queue_count.checked_add(1).unwrap();
                         to_execute_substage.send(ExecutablePayload(ee)).unwrap();
                     }
-                    trace!("schedule_once [C] id_{:016x} ch(prev: {}, exec: {}|{}), runnnable: {}, contended: {}, (immediate+provisional)/max: ({}+{})/{} uncontended: {} stuck: {} completed: {}!", random_id, from_prev.len(), to_execute_substage.len(), from_exec.len(), runnable_queue.task_count(), contended_count, executing_queue_count, provisioning_tracker_count, max_executing_queue_count, address_book.uncontended_task_ids.len(), address_book.stuck_tasks.len(), completed_count);
+                    trace!("schedule_once id_{:016x} [C] ch(prev: {}, exec: {}|{}), runnnable: {}, contended: {}, (immediate+provisional)/max: ({}+{})/{} uncontended: {} stuck: {} completed: {}!", random_id, from_prev.len(), to_execute_substage.len(), from_exec.len(), runnable_queue.task_count(), contended_count, executing_queue_count, provisioning_tracker_count, max_executing_queue_count, address_book.uncontended_task_ids.len(), address_book.stuck_tasks.len(), completed_count);
                     if start.elapsed() > std::time::Duration::from_millis(150) {
                         start = std::time::Instant::now();
                         info!("schedule_once:interval id_{:016x} ch(prev: {}, exec: {}|{}), runnnable: {}, contended: {}, (immediate+provisional)/max: ({}+{})/{} uncontended: {} stuck: {} completed: {}!", random_id, from_prev.len(), to_execute_substage.len(), from_exec.len(), runnable_queue.task_count(), contended_count, executing_queue_count, provisioning_tracker_count, max_executing_queue_count, address_book.uncontended_task_ids.len(), address_book.stuck_tasks.len(), completed_count);

From 968bd83c0ba198085086f286b36acf7d082bf216 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 8 Sep 2022 15:02:09 +0900
Subject: [PATCH 2294/3199] save

---
 scheduler/src/lib.rs | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 282fec558c531d..4a5da2c34d97c1 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -906,10 +906,12 @@ impl ScheduleStage {
                     Some((TaskSource::Runnable, t))
                     */
                 } else if uw > weight_from_runnable {
-                    trace!("select: contended > runnnable");
                     if runnable_exclusive {
-                        None
+                        trace!("select: contended > runnnable, runnable_exclusive");
+                        let t = heaviest_runnable_entry.remove();
+                        Some((TaskSource::Runnable, t))
                     } else {
+                        trace!("select: contended > runnnable, !runnable_exclusive)");
                         let t = weight_from_contended.remove();
                         Some((TaskSource::Contended, t))
                     }

From 56d09a745e4d0a7bcb067145aa37824d07550fb5 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 8 Sep 2022 15:04:40 +0900
Subject: [PATCH 2295/3199] save

---
 scheduler/src/lib.rs | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 4a5da2c34d97c1..179bb02c50e670 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1496,12 +1496,12 @@ impl ScheduleStage {
                         &mut queue_clock,
                         &mut execute_clock,
                         &mut provisioning_tracker_count,
-                        true
+                        false,
                     ) {
                         executing_queue_count = executing_queue_count.checked_add(1).unwrap();
                         to_execute_substage.send(ExecutablePayload(ee)).unwrap();
                     }
-                    trace!("schedule_once id_{:016x} [R] ch(prev: {}, exec: {}|{}), runnnable: {}, contended: {}, (immediate+provisional)/max: ({}+{})/{} uncontended: {} stuck: {} completed: {}!", random_id, from_prev.len(), to_execute_substage.len(), from_exec.len(), runnable_queue.task_count(), contended_count, executing_queue_count, provisioning_tracker_count, max_executing_queue_count, address_book.uncontended_task_ids.len(), address_book.stuck_tasks.len(), completed_count);
+                    trace!("schedule_once id_{:016x} [C] ch(prev: {}, exec: {}|{}), runnnable: {}, contended: {}, (immediate+provisional)/max: ({}+{})/{} uncontended: {} stuck: {} completed: {}!", random_id, from_prev.len(), to_execute_substage.len(), from_exec.len(), runnable_queue.task_count(), contended_count, executing_queue_count, provisioning_tracker_count, max_executing_queue_count, address_book.uncontended_task_ids.len(), address_book.stuck_tasks.len(), completed_count);
                     if start.elapsed() > std::time::Duration::from_millis(150) {
                         start = std::time::Instant::now();
                         info!("schedule_once:interval id_{:016x} ch(prev: {}, exec: {}|{}), runnnable: {}, contended: {}, (immediate+provisional)/max: ({}+{})/{} uncontended: {} stuck: {} completed: {}!", random_id, from_prev.len(), to_execute_substage.len(), from_exec.len(), runnable_queue.task_count(), contended_count, executing_queue_count, provisioning_tracker_count, max_executing_queue_count, address_book.uncontended_task_ids.len(), address_book.stuck_tasks.len(), completed_count);
@@ -1522,12 +1522,12 @@ impl ScheduleStage {
                         &mut queue_clock,
                         &mut execute_clock,
                         &mut provisioning_tracker_count,
-                        false,
+                        true
                     ) {
                         executing_queue_count = executing_queue_count.checked_add(1).unwrap();
                         to_execute_substage.send(ExecutablePayload(ee)).unwrap();
                     }
-                    trace!("schedule_once id_{:016x} [C] ch(prev: {}, exec: {}|{}), runnnable: {}, contended: {}, (immediate+provisional)/max: ({}+{})/{} uncontended: {} stuck: {} completed: {}!", random_id, from_prev.len(), to_execute_substage.len(), from_exec.len(), runnable_queue.task_count(), contended_count, executing_queue_count, provisioning_tracker_count, max_executing_queue_count, address_book.uncontended_task_ids.len(), address_book.stuck_tasks.len(), completed_count);
+                    trace!("schedule_once id_{:016x} [R] ch(prev: {}, exec: {}|{}), runnnable: {}, contended: {}, (immediate+provisional)/max: ({}+{})/{} uncontended: {} stuck: {} completed: {}!", random_id, from_prev.len(), to_execute_substage.len(), from_exec.len(), runnable_queue.task_count(), contended_count, executing_queue_count, provisioning_tracker_count, max_executing_queue_count, address_book.uncontended_task_ids.len(), address_book.stuck_tasks.len(), completed_count);
                     if start.elapsed() > std::time::Duration::from_millis(150) {
                         start = std::time::Instant::now();
                         info!("schedule_once:interval id_{:016x} ch(prev: {}, exec: {}|{}), runnnable: {}, contended: {}, (immediate+provisional)/max: ({}+{})/{} uncontended: {} stuck: {} completed: {}!", random_id, from_prev.len(), to_execute_substage.len(), from_exec.len(), runnable_queue.task_count(), contended_count, executing_queue_count, provisioning_tracker_count, max_executing_queue_count, address_book.uncontended_task_ids.len(), address_book.stuck_tasks.len(), completed_count);

From 37a2df6da6e28688baac68de426220477ed3c065 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 8 Sep 2022 15:22:20 +0900
Subject: [PATCH 2296/3199] save

---
 scheduler/src/lib.rs | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 179bb02c50e670..750c2e84da0473 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1507,8 +1507,7 @@ impl ScheduleStage {
                         info!("schedule_once:interval id_{:016x} ch(prev: {}, exec: {}|{}), runnnable: {}, contended: {}, (immediate+provisional)/max: ({}+{})/{} uncontended: {} stuck: {} completed: {}!", random_id, from_prev.len(), to_execute_substage.len(), from_exec.len(), runnable_queue.task_count(), contended_count, executing_queue_count, provisioning_tracker_count, max_executing_queue_count, address_book.uncontended_task_ids.len(), address_book.stuck_tasks.len(), completed_count);
                     }
                 }
-                let executing_like_count = executing_queue_count + provisioning_tracker_count;
-                if executing_like_count < max_executing_queue_count {
+                while executing_queue_count + provisioning_tracker_count < max_executing_queue_count {
                     let prefer_immediate = provisioning_tracker_count / 4 > executing_queue_count;
 
                     if let Some(ee) = Self::schedule_next_execution(
@@ -1531,6 +1530,12 @@ impl ScheduleStage {
                     if start.elapsed() > std::time::Duration::from_millis(150) {
                         start = std::time::Instant::now();
                         info!("schedule_once:interval id_{:016x} ch(prev: {}, exec: {}|{}), runnnable: {}, contended: {}, (immediate+provisional)/max: ({}+{})/{} uncontended: {} stuck: {} completed: {}!", random_id, from_prev.len(), to_execute_substage.len(), from_exec.len(), runnable_queue.task_count(), contended_count, executing_queue_count, provisioning_tracker_count, max_executing_queue_count, address_book.uncontended_task_ids.len(), address_book.stuck_tasks.len(), completed_count);
+                    } else {
+                        break;
+                    }
+                    if !from_exec.is_empty() {
+                        trace!("abort aggressive readalbe queue processing due to non-empty from_exec");
+                        break;
                     }
                 }
 

From 35d800a79cbc1eb24b3c1af00b45638b115f27be Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 8 Sep 2022 16:55:05 +0900
Subject: [PATCH 2297/3199] save

---
 scheduler/src/lib.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 750c2e84da0473..d6815adabfd5b9 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1483,7 +1483,7 @@ impl ScheduleStage {
             loop {
                 let executing_like_count = executing_queue_count + provisioning_tracker_count;
                 if executing_like_count < max_executing_queue_count {
-                    let prefer_immediate = provisioning_tracker_count / 4 > executing_queue_count;
+                    let prefer_immediate = true; //provisioning_tracker_count / 4 > executing_queue_count;
 
                     if let Some(ee) = Self::schedule_next_execution(
                         ast,
@@ -1508,7 +1508,7 @@ impl ScheduleStage {
                     }
                 }
                 while executing_queue_count + provisioning_tracker_count < max_executing_queue_count {
-                    let prefer_immediate = provisioning_tracker_count / 4 > executing_queue_count;
+                    let prefer_immediate = true; //provisioning_tracker_count / 4 > executing_queue_count;
 
                     if let Some(ee) = Self::schedule_next_execution(
                         ast,

From 82c585be7eda0359c71f2b0e7851a3ec0bd7c191 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 8 Sep 2022 20:40:30 +0900
Subject: [PATCH 2298/3199] save

---
 scheduler/src/lib.rs | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index d6815adabfd5b9..54ac5913bae613 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -237,18 +237,20 @@ pub struct Page {
     next_usage: Usage,
     provisional_task_ids: Vec<triomphe::Arc<ProvisioningTracker>>,
     cu: CU,
+    address_str: String,
     //loaded account from Accounts db
     //comulative_cu for qos; i.e. track serialized cumulative keyed by addresses and bail out block
     //producing as soon as any one of cu from the executing thread reaches to the limit
 }
 
 impl Page {
-    fn new(current_usage: Usage) -> Self {
+    fn new(address: &Pubkey, current_usage: Usage) -> Self {
         Self {
             current_usage,
             next_usage: Usage::Unused,
             provisional_task_ids: Default::default(),
             cu: Default::default(),
+            address: format!("{}", address),
         }
     }
 

From d86824ee2bd65f2d94df5728ffa8b2d1231c171f Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 8 Sep 2022 20:40:43 +0900
Subject: [PATCH 2299/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 54ac5913bae613..14333440271165 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -250,7 +250,7 @@ impl Page {
             next_usage: Usage::Unused,
             provisional_task_ids: Default::default(),
             cu: Default::default(),
-            address: format!("{}", address),
+            address_str: format!("{}", address),
         }
     }
 

From d2ec3e2484c2f06eedb8c62d9813b87a3b627b98 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 8 Sep 2022 20:40:58 +0900
Subject: [PATCH 2300/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 14333440271165..a67b3a1ed440f8 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -485,7 +485,7 @@ impl Preloader {
     pub fn load(&self, address: Pubkey) -> PageRc {
         PageRc::clone(&self.book.entry(address).or_insert_with(|| {
             PageRc(PageRcInner::new((
-                core::cell::RefCell::new(Page::new(Usage::unused())),
+                core::cell::RefCell::new(Page::new(&address, Usage::unused())),
                 Default::default(),
             )))
         }))

From ae67c7231559a07b91e2b658a3b590ab2731300f Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 8 Sep 2022 20:44:13 +0900
Subject: [PATCH 2301/3199] save

---
 scheduler/src/lib.rs | 1 +
 1 file changed, 1 insertion(+)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index a67b3a1ed440f8..e23f016d5d10c8 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -818,6 +818,7 @@ fn attempt_lock_for_execution<'a, AST: AtScheduleThread>(
         match attempt.status {
             LockStatus::Succeded => {}
             LockStatus::Failed => {
+                trace!("lock failed: {}/{:?}", attempt.target.address_str, attempt.requested_usage);
                 unlockable_count += 1;
             }
             LockStatus::Provisional => {

From bcac16afd74d37cfe2e2b5cc27c7636ac57264eb Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 8 Sep 2022 20:44:32 +0900
Subject: [PATCH 2302/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index e23f016d5d10c8..952b27758c1128 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -818,7 +818,7 @@ fn attempt_lock_for_execution<'a, AST: AtScheduleThread>(
         match attempt.status {
             LockStatus::Succeded => {}
             LockStatus::Failed => {
-                trace!("lock failed: {}/{:?}", attempt.target.address_str, attempt.requested_usage);
+                trace!("lock failed: {}/{:?}", attempt.target,page_ref().address_str, attempt.requested_usage);
                 unlockable_count += 1;
             }
             LockStatus::Provisional => {

From d92637ffb4900db727893aaf21729db1baf2a1c8 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 8 Sep 2022 20:44:43 +0900
Subject: [PATCH 2303/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 952b27758c1128..7f950bc7490119 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -818,7 +818,7 @@ fn attempt_lock_for_execution<'a, AST: AtScheduleThread>(
         match attempt.status {
             LockStatus::Succeded => {}
             LockStatus::Failed => {
-                trace!("lock failed: {}/{:?}", attempt.target,page_ref().address_str, attempt.requested_usage);
+                trace!("lock failed: {}/{:?}", attempt.target.page_ref().address_str, attempt.requested_usage);
                 unlockable_count += 1;
             }
             LockStatus::Provisional => {

From 12f4da9c50b070ca5f2bc85a21c82c9e2fa5ff9b Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 8 Sep 2022 20:44:59 +0900
Subject: [PATCH 2304/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 7f950bc7490119..6476c2a841706c 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -818,7 +818,7 @@ fn attempt_lock_for_execution<'a, AST: AtScheduleThread>(
         match attempt.status {
             LockStatus::Succeded => {}
             LockStatus::Failed => {
-                trace!("lock failed: {}/{:?}", attempt.target.page_ref().address_str, attempt.requested_usage);
+                trace!("lock failed: {}/{:?}", attempt.page_ref().address_str, attempt.requested_usage);
                 unlockable_count += 1;
             }
             LockStatus::Provisional => {

From c3accc57b853bf11affa308013ddd377e1191fd8 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 8 Sep 2022 20:45:15 +0900
Subject: [PATCH 2305/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 6476c2a841706c..eb2cf840c68b6a 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -818,7 +818,7 @@ fn attempt_lock_for_execution<'a, AST: AtScheduleThread>(
         match attempt.status {
             LockStatus::Succeded => {}
             LockStatus::Failed => {
-                trace!("lock failed: {}/{:?}", attempt.page_ref().address_str, attempt.requested_usage);
+                trace!("lock failed: {}/{:?}", attempt.page_mut().address_str, attempt.requested_usage);
                 unlockable_count += 1;
             }
             LockStatus::Provisional => {

From 07afa2f538e9aa47f20132f429549c62801cea4c Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 8 Sep 2022 20:45:25 +0900
Subject: [PATCH 2306/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index eb2cf840c68b6a..e6e8d7bffcaa49 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -818,7 +818,7 @@ fn attempt_lock_for_execution<'a, AST: AtScheduleThread>(
         match attempt.status {
             LockStatus::Succeded => {}
             LockStatus::Failed => {
-                trace!("lock failed: {}/{:?}", attempt.page_mut().address_str, attempt.requested_usage);
+                trace!("lock failed: {}/{:?}", attempt.target.page_mut().address_str, attempt.requested_usage);
                 unlockable_count += 1;
             }
             LockStatus::Provisional => {

From 3a3ba367b1aeac8642efa1be0715a9a03717b8d0 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 8 Sep 2022 20:45:34 +0900
Subject: [PATCH 2307/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index e6e8d7bffcaa49..12d1e1727cef9a 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -818,7 +818,7 @@ fn attempt_lock_for_execution<'a, AST: AtScheduleThread>(
         match attempt.status {
             LockStatus::Succeded => {}
             LockStatus::Failed => {
-                trace!("lock failed: {}/{:?}", attempt.target.page_mut().address_str, attempt.requested_usage);
+                trace!("lock failed: {}/{:?}", attempt.target.page_mut(ast).address_str, attempt.requested_usage);
                 unlockable_count += 1;
             }
             LockStatus::Provisional => {

From 9c792c84e009c1d540e76b36d19f3cb46c4e82ef Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 8 Sep 2022 21:14:13 +0900
Subject: [PATCH 2308/3199] save

---
 scheduler/src/lib.rs | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 12d1e1727cef9a..e6cc13938046d1 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -233,11 +233,12 @@ impl TaskIds {
 
 #[derive(Debug)]
 pub struct Page {
+    address_str: String,
     current_usage: Usage,
     next_usage: Usage,
     provisional_task_ids: Vec<triomphe::Arc<ProvisioningTracker>>,
     cu: CU,
-    address_str: String,
+    contended_write_task_count: usize,
     //loaded account from Accounts db
     //comulative_cu for qos; i.e. track serialized cumulative keyed by addresses and bail out block
     //producing as soon as any one of cu from the executing thread reaches to the limit
@@ -246,11 +247,12 @@ pub struct Page {
 impl Page {
     fn new(address: &Pubkey, current_usage: Usage) -> Self {
         Self {
+            address_str: format!("{}", address),
             current_usage,
             next_usage: Usage::Unused,
             provisional_task_ids: Default::default(),
             cu: Default::default(),
-            address_str: format!("{}", address),
+            contended_write_task_count: Default::default(),
         }
     }
 

From 0e3cbb959654d4c9342753f8e22463c536bb228a Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 8 Sep 2022 21:17:14 +0900
Subject: [PATCH 2309/3199] save

---
 scheduler/src/lib.rs | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index e6cc13938046d1..d8ef7cd1eb17aa 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -590,14 +590,14 @@ impl Task {
         &self,
         _ast: AST,
     ) -> std::cell::RefMut<'_, Vec<LockAttempt>> {
-        self.tx.1 .0.borrow_mut()
+        self.tx.1.0.borrow_mut()
     }
 
     fn lock_attempts_not_mut<NAST: NotAtScheduleThread>(
         &self,
         _nast: NAST,
     ) -> std::cell::Ref<'_, Vec<LockAttempt>> {
-        self.tx.1 .0.borrow()
+        self.tx.1.0.borrow()
     }
 
     fn update_busiest_page_cu(&self, cu: CU) {
@@ -732,6 +732,10 @@ impl Task {
     ) {
         for lock_attempt in this.lock_attempts_mut(ast).iter() {
             lock_attempt.target_contended_unique_weights().insert_task(this.unique_weight, Task::clone_in_queue(this));
+            if lock_attempt.requested_usage == RequestedUsage::Writable {
+                lock_attempt.target.page_mut().contended_write_task_count = 
+                    lock_attempt.target.page_mut().contended_write_task_count.checked_add(1).unwrap();
+            }
         }
         //let a = Task::clone_in_queue(this);
         //task_sender

From f323f5e75911a87484f0eb951f0cc656bb4d0f9e Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 8 Sep 2022 21:17:43 +0900
Subject: [PATCH 2310/3199] save

---
 scheduler/src/lib.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index d8ef7cd1eb17aa..a9d31739547103 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -733,8 +733,8 @@ impl Task {
         for lock_attempt in this.lock_attempts_mut(ast).iter() {
             lock_attempt.target_contended_unique_weights().insert_task(this.unique_weight, Task::clone_in_queue(this));
             if lock_attempt.requested_usage == RequestedUsage::Writable {
-                lock_attempt.target.page_mut().contended_write_task_count = 
-                    lock_attempt.target.page_mut().contended_write_task_count.checked_add(1).unwrap();
+                let mut page = lock_attempt.target.page_mut(ast);
+                page.contended_write_task_count = page.contended_write_task_count.checked_add(1).unwrap();
             }
         }
         //let a = Task::clone_in_queue(this);

From f40ec57e8d4d17801e4c197a8c47c718d81a0c76 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 8 Sep 2022 21:18:08 +0900
Subject: [PATCH 2311/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index a9d31739547103..125d7e86ab8daf 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -195,7 +195,7 @@ impl Usage {
     }
 }
 
-#[derive(Clone, Copy, Debug)]
+#[derive(Clone, Copy, Debug, PartialEq, Eq)]
 pub enum RequestedUsage {
     Readonly,
     Writable,

From aa7fd1c63212ca1d04506705f840ea1ee0ee06d9 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 8 Sep 2022 21:22:19 +0900
Subject: [PATCH 2312/3199] save

---
 scheduler/src/lib.rs | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 125d7e86ab8daf..bfb2b90fd10411 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -102,6 +102,11 @@ impl ExecutionEnvironment {
                     lock_attempt.heaviest_uncontended = Some(task);
                     ()
                 });
+
+            if lock_attempt.requested_usage == RequestedUsage::Writable {
+                let page = lock_attampt.target.page_mut();
+                page.contended_write_task_count = page.contended_write_task_count.checked_sub(1).unwrap();
+            }
         }
     }
 
@@ -220,7 +225,7 @@ impl TaskIds {
     #[inline(never)]
     pub fn remove_task(&self, u: &TaskId) {
         let removed_entry = self.task_ids.remove(u);
-        //assert!(removed_entry.is_some());
+        assert!(removed_entry.is_some());
     }
 
     #[inline(never)]
@@ -732,6 +737,7 @@ impl Task {
     ) {
         for lock_attempt in this.lock_attempts_mut(ast).iter() {
             lock_attempt.target_contended_unique_weights().insert_task(this.unique_weight, Task::clone_in_queue(this));
+
             if lock_attempt.requested_usage == RequestedUsage::Writable {
                 let mut page = lock_attempt.target.page_mut(ast);
                 page.contended_write_task_count = page.contended_write_task_count.checked_add(1).unwrap();
@@ -1439,6 +1445,7 @@ impl ScheduleStage {
                             lock_attempt
                                 .target_contended_unique_weights()
                                 .insert_task(task.unique_weight, Task::clone_in_queue(&task));
+                            todo!("contended_write_task_count!");
                         }
                     }
                     assert_eq!(task_receiver.len(), 0);

From 720a047c39114c841b79e8e010899fe0ffb70c86 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 8 Sep 2022 21:22:31 +0900
Subject: [PATCH 2313/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index bfb2b90fd10411..b9af030c0bb63b 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -104,7 +104,7 @@ impl ExecutionEnvironment {
                 });
 
             if lock_attempt.requested_usage == RequestedUsage::Writable {
-                let page = lock_attampt.target.page_mut();
+                let page = lock_attempt.target.page_mut();
                 page.contended_write_task_count = page.contended_write_task_count.checked_sub(1).unwrap();
             }
         }

From cc1fc26fde882ff10166848625a7559f932e56de Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 8 Sep 2022 21:23:15 +0900
Subject: [PATCH 2314/3199] save

---
 scheduler/src/lib.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index b9af030c0bb63b..0a9ddd911792f5 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -52,7 +52,7 @@ impl ExecutionEnvironment {
     //}
     //
     #[inline(never)]
-    pub fn reindex_with_address_book(&mut self) {
+    pub fn reindex_with_address_book<AST: AtScheduleThread>(&mut self, ast: AST) {
         assert!(!self.is_reindexed());
         self.is_reindexed = true;
 
@@ -104,7 +104,7 @@ impl ExecutionEnvironment {
                 });
 
             if lock_attempt.requested_usage == RequestedUsage::Writable {
-                let page = lock_attempt.target.page_mut();
+                let page = lock_attempt.target.page_mut(ast);
                 page.contended_write_task_count = page.contended_write_task_count.checked_sub(1).unwrap();
             }
         }

From fffe1272057eb210bf668b451f9a49025d07ae67 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 8 Sep 2022 21:23:24 +0900
Subject: [PATCH 2315/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 0a9ddd911792f5..4c8f030db91f96 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1302,7 +1302,7 @@ impl ScheduleStage {
     ) {
         // do par()-ly?
 
-        ee.reindex_with_address_book();
+        ee.reindex_with_address_book(ast);
         assert!(ee.is_reindexed());
 
         ee.task.record_commit_time(*commit_clock);

From f1462150b9ec3fc795a5f7c9d0f4a7ae2530fabe Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 8 Sep 2022 21:23:36 +0900
Subject: [PATCH 2316/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 4c8f030db91f96..833a9cf5213d2e 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -104,7 +104,7 @@ impl ExecutionEnvironment {
                 });
 
             if lock_attempt.requested_usage == RequestedUsage::Writable {
-                let page = lock_attempt.target.page_mut(ast);
+                let mut page = lock_attempt.target.page_mut(ast);
                 page.contended_write_task_count = page.contended_write_task_count.checked_sub(1).unwrap();
             }
         }

From a182bba0ee93e43d27b3dcee36b218cd71ed19c6 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 8 Sep 2022 21:23:51 +0900
Subject: [PATCH 2317/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 833a9cf5213d2e..f07b88d0907612 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -52,7 +52,7 @@ impl ExecutionEnvironment {
     //}
     //
     #[inline(never)]
-    pub fn reindex_with_address_book<AST: AtScheduleThread>(&mut self, ast: AST) {
+    fn reindex_with_address_book<AST: AtScheduleThread>(&mut self, ast: AST) {
         assert!(!self.is_reindexed());
         self.is_reindexed = true;
 

From beb21d0df9b6855f52db63890401d08822076b9a Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 8 Sep 2022 21:25:27 +0900
Subject: [PATCH 2318/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index f07b88d0907612..1342bed251faaa 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -735,7 +735,7 @@ impl Task {
         this: &TaskInQueue,
         task_sender: &crossbeam_channel::Sender<(TaskInQueue, Vec<LockAttempt>)>,
     ) {
-        for lock_attempt in this.lock_attempts_mut(ast).iter() {
+        for lock_attempt in this.lock_attempts_mut(ast) {
             lock_attempt.target_contended_unique_weights().insert_task(this.unique_weight, Task::clone_in_queue(this));
 
             if lock_attempt.requested_usage == RequestedUsage::Writable {

From 8637865a2ed228d25f4f19839ed614a9596b2f5f Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 8 Sep 2022 21:25:39 +0900
Subject: [PATCH 2319/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 1342bed251faaa..db02c265b3a48e 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -735,7 +735,7 @@ impl Task {
         this: &TaskInQueue,
         task_sender: &crossbeam_channel::Sender<(TaskInQueue, Vec<LockAttempt>)>,
     ) {
-        for lock_attempt in this.lock_attempts_mut(ast) {
+        for lock_attempt in &this.lock_attempts_mut(ast) {
             lock_attempt.target_contended_unique_weights().insert_task(this.unique_weight, Task::clone_in_queue(this));
 
             if lock_attempt.requested_usage == RequestedUsage::Writable {

From 4071be7cd7dfa68c27a4d6858bfd34665723b2f2 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 8 Sep 2022 21:25:48 +0900
Subject: [PATCH 2320/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index db02c265b3a48e..a1d2c566c4df7e 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -735,7 +735,7 @@ impl Task {
         this: &TaskInQueue,
         task_sender: &crossbeam_channel::Sender<(TaskInQueue, Vec<LockAttempt>)>,
     ) {
-        for lock_attempt in &this.lock_attempts_mut(ast) {
+        for lock_attempt in &*this.lock_attempts_mut(ast) {
             lock_attempt.target_contended_unique_weights().insert_task(this.unique_weight, Task::clone_in_queue(this));
 
             if lock_attempt.requested_usage == RequestedUsage::Writable {

From b94f035252ce7193d81c702157c4a3d8cff590bb Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 8 Sep 2022 21:32:53 +0900
Subject: [PATCH 2321/3199] save

---
 scheduler/src/lib.rs | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index a1d2c566c4df7e..7e2f425f9f2285 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -327,8 +327,13 @@ impl AddressBook {
         unique_weight: &UniqueWeight,
         attempt: &mut LockAttempt,
     ) -> CU {
-        let strictly_lockable_for_replay = attempt.target_contended_unique_weights().task_ids.is_empty() ||
-            attempt.target_contended_unique_weights().task_ids.back().unwrap().key() == unique_weight;
+        let strictly_lockable_for_replay = if attempt.target_contended_unique_weights().task_ids.is_empty() {
+            true
+        } else if attempt.target_contended_unique_weights().task_ids.back().unwrap().key() == unique_weight {
+            true
+        } else if attempt.requested_usage == Readonly && attempt.target.page_mut(ast).contended_write_task_count == 0 {
+            true
+        }
 
         if !strictly_lockable_for_replay {
             attempt.status = LockStatus::Failed;

From 2fbd6d229a39e67cf0129f168880c79f40b13309 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 8 Sep 2022 21:33:13 +0900
Subject: [PATCH 2322/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 7e2f425f9f2285..4eedb58f6f6e5e 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -331,7 +331,7 @@ impl AddressBook {
             true
         } else if attempt.target_contended_unique_weights().task_ids.back().unwrap().key() == unique_weight {
             true
-        } else if attempt.requested_usage == Readonly && attempt.target.page_mut(ast).contended_write_task_count == 0 {
+        } else if attempt.requested_usage == RequestedUsage::Readonly && attempt.target.page_mut(ast).contended_write_task_count == 0 {
             true
         }
 

From 9dd61dfe6f28b8e031cfc357581406267190739e Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 8 Sep 2022 21:33:24 +0900
Subject: [PATCH 2323/3199] save

---
 scheduler/src/lib.rs | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 4eedb58f6f6e5e..8b7de3a6bccb93 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -333,6 +333,8 @@ impl AddressBook {
             true
         } else if attempt.requested_usage == RequestedUsage::Readonly && attempt.target.page_mut(ast).contended_write_task_count == 0 {
             true
+        } else {
+            false
         }
 
         if !strictly_lockable_for_replay {

From dd2b182c784092936114eb3d7972f2163304da4d Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 8 Sep 2022 21:33:31 +0900
Subject: [PATCH 2324/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 8b7de3a6bccb93..9ad9d843678e3a 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -335,7 +335,7 @@ impl AddressBook {
             true
         } else {
             false
-        }
+        };
 
         if !strictly_lockable_for_replay {
             attempt.status = LockStatus::Failed;

From 60b27cfb90f678c2965f3d73c5dfec84c1adf5ff Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 8 Sep 2022 21:40:43 +0900
Subject: [PATCH 2325/3199] save

---
 ledger-tool/src/main.rs | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/ledger-tool/src/main.rs b/ledger-tool/src/main.rs
index 472a61ccafb9e0..9fe13087bce6e3 100644
--- a/ledger-tool/src/main.rs
+++ b/ledger-tool/src/main.rs
@@ -340,7 +340,8 @@ fn output_slot(
                             .send(solana_scheduler::Multiplexed::FromExecute(ee))
                             .unwrap();
                         */
-                        ee.reindex_with_address_book();
+                        // ee.reindex_with_address_book();
+                        todo!("contended_write_task_count");
                         post_execute_env_sender.send(solana_scheduler::UnlockablePayload(ee)).unwrap();
                     }
                 })

From 8ca9a433e1a03f06ffd7f3ed2adc9dcb29e9868e Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 8 Sep 2022 21:58:16 +0900
Subject: [PATCH 2326/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 9ad9d843678e3a..73a6877264531a 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -103,7 +103,7 @@ impl ExecutionEnvironment {
                     ()
                 });
 
-            if lock_attempt.requested_usage == RequestedUsage::Writable {
+            if should_remove && lock_attempt.requested_usage == RequestedUsage::Writable {
                 let mut page = lock_attempt.target.page_mut(ast);
                 page.contended_write_task_count = page.contended_write_task_count.checked_sub(1).unwrap();
             }

From 77b58c57155f85750be851bf276bae5aed12283b Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 9 Sep 2022 15:24:24 +0900
Subject: [PATCH 2327/3199] save

---
 runtime/src/bank.rs  | 14 +++++++-------
 scheduler/src/lib.rs | 36 ++++++++++++++++++------------------
 2 files changed, 25 insertions(+), 25 deletions(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 707fecac81d7f5..67ec5db54d329b 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1240,8 +1240,8 @@ impl Default for Scheduler {
         let preloader = Arc::new(address_book.preloader());
         let (transaction_sender, transaction_receiver) = crossbeam_channel::unbounded();
         let (scheduled_ee_sender, scheduled_ee_receiver) = crossbeam_channel::unbounded();
-        let (completed_ee_sender, completed_ee_receiver) = crossbeam_channel::unbounded();
-        let (droppable_ee_sender, droppable_ee_receiver) = crossbeam_channel::unbounded();
+        let (processed_ee_sender, processed_ee_receiver) = crossbeam_channel::unbounded();
+        let (retired_ee_sender, retired_ee_receiver) = crossbeam_channel::unbounded();
 
         let bank = Arc::new(std::sync::RwLock::new(None::<Arc<Bank>>));
 
@@ -1254,7 +1254,7 @@ impl Default for Scheduler {
         let disable_clock_asserts = std::env::var("DISABLE_CLOCK_ASSERTS").is_ok();
 
         let executing_thread_handles = (0..executing_thread_count).map(|thx| {
-            let (scheduled_ee_receiver, completed_ee_sender) = (scheduled_ee_receiver.clone(), completed_ee_sender.clone());
+            let (scheduled_ee_receiver, processed_ee_sender) = (scheduled_ee_receiver.clone(), processed_ee_sender.clone());
             let bank = bank.clone();
             let mut execute_time = 0;
 
@@ -1332,7 +1332,7 @@ impl Default for Scheduler {
         let errors_in_collector_thread = Arc::clone(&errors);
 
         let error_collector_thread_handle = std::thread::Builder::new().name(format!("solScErrCol{:02}", 0)).spawn(move || {
-            while let Ok(solana_scheduler::DroppablePayload(mut ee)) = droppable_ee_receiver.recv() {
+            while let Ok(solana_scheduler::ExaminablePayload(mut ee)) = retired_ee_receiver.recv() {
                 if ee.is_aborted() {
                     warn!(
                         "scheduler: Unexpected validator error: {:?}, transaction: {:?}",
@@ -1359,12 +1359,12 @@ impl Default for Scheduler {
                 &mut address_book,
                 &transaction_receiver,
                 &scheduled_ee_sender,
-                &completed_ee_receiver,
-                Some(&droppable_ee_sender),
+                &processed_ee_receiver,
+                Some(&retired_ee_sender),
             );
             drop(transaction_receiver);
             drop(scheduled_ee_sender);
-            drop(completed_ee_receiver);
+            drop(processed_ee_receiver);
 
             Ok(())
         }).unwrap();
diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 73a6877264531a..dbd216e6a17b99 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1300,7 +1300,7 @@ impl ScheduleStage {
     }
 
     #[inline(never)]
-    fn commit_completed_execution<AST: AtScheduleThread>(
+    fn commit_processed_execution<AST: AtScheduleThread>(
         ast: AST,
         ee: &mut ExecutionEnvironment,
         address_book: &mut AddressBook,
@@ -1390,7 +1390,7 @@ impl ScheduleStage {
         mut from_prev: &'a crossbeam_channel::Receiver<SchedulablePayload>,
         to_execute_substage: &crossbeam_channel::Sender<ExecutablePayload>,
         from_exec: &crossbeam_channel::Receiver<UnlockablePayload>,
-        maybe_to_next_stage: Option<&crossbeam_channel::Sender<DroppablePayload>>, // assume nonblocking
+        maybe_to_next_stage: Option<&crossbeam_channel::Sender<ExaminablePayload>>, // assume nonblocking
         never: &'a crossbeam_channel::Receiver<SchedulablePayload>,
     ) {
         let random_id = rand::thread_rng().gen::<u64>();
@@ -1403,10 +1403,10 @@ impl ScheduleStage {
         let mut queue_clock = 0;
         let mut execute_clock = 0;
         let mut commit_clock = 0;
-        let mut completed_count = 0_usize;
+        let mut processed_count = 0_usize;
         assert!(max_executing_queue_count > 0);
 
-        let (ee_sender, ee_receiver) = crossbeam_channel::unbounded::<DroppablePayload>();
+        let (ee_sender, ee_receiver) = crossbeam_channel::unbounded::<ExaminablePayload>();
 
         let (to_next_stage, maybe_reaper_thread_handle) = if let Some(to_next_stage) = maybe_to_next_stage {
             (to_next_stage, None)
@@ -1419,7 +1419,7 @@ impl ScheduleStage {
                     unsafe impl NotAtScheduleThread for NotAtTopOfScheduleThread {}
                     let nast = NotAtTopOfScheduleThread;
 
-                    while let Ok(DroppablePayload(mut a)) = ee_receiver.recv() {
+                    while let Ok(ExaminablePayload(mut a)) = ee_receiver.recv() {
                         assert!(a.task.lock_attempts_not_mut(nast).is_empty());
                         //assert!(a.task.sequence_time() != usize::max_value());
                         //let lock_attempts = std::mem::take(&mut a.lock_attempts);
@@ -1469,9 +1469,9 @@ impl ScheduleStage {
                recv(from_exec) -> maybe_from_exec => {
                    if let Ok(UnlockablePayload(mut processed_execution_environment)) = maybe_from_exec {
                        executing_queue_count = executing_queue_count.checked_sub(1).unwrap();
-                       completed_count = completed_count.checked_add(1).unwrap();
-                       Self::commit_completed_execution(ast, &mut processed_execution_environment, address_book, &mut commit_clock, &mut provisioning_tracker_count);
-                       to_next_stage.send(DroppablePayload(processed_execution_environment)).unwrap();
+                       processed_count = processed_count.checked_add(1).unwrap();
+                       Self::commit_processed_execution(ast, &mut processed_execution_environment, address_book, &mut commit_clock, &mut provisioning_tracker_count);
+                       to_next_stage.send(ExaminablePayload(processed_execution_environment)).unwrap();
                    } else {
                        assert_eq!(from_exec.len(), 0);
                        from_exec_disconnected |= true;
@@ -1524,10 +1524,10 @@ impl ScheduleStage {
                         executing_queue_count = executing_queue_count.checked_add(1).unwrap();
                         to_execute_substage.send(ExecutablePayload(ee)).unwrap();
                     }
-                    trace!("schedule_once id_{:016x} [C] ch(prev: {}, exec: {}|{}), runnnable: {}, contended: {}, (immediate+provisional)/max: ({}+{})/{} uncontended: {} stuck: {} completed: {}!", random_id, from_prev.len(), to_execute_substage.len(), from_exec.len(), runnable_queue.task_count(), contended_count, executing_queue_count, provisioning_tracker_count, max_executing_queue_count, address_book.uncontended_task_ids.len(), address_book.stuck_tasks.len(), completed_count);
+                    trace!("schedule_once id_{:016x} [C] ch(prev: {}, exec: {}|{}), runnnable: {}, contended: {}, (immediate+provisional)/max: ({}+{})/{} uncontended: {} stuck: {} processed: {}!", random_id, from_prev.len(), to_execute_substage.len(), from_exec.len(), runnable_queue.task_count(), contended_count, executing_queue_count, provisioning_tracker_count, max_executing_queue_count, address_book.uncontended_task_ids.len(), address_book.stuck_tasks.len(), processed_count);
                     if start.elapsed() > std::time::Duration::from_millis(150) {
                         start = std::time::Instant::now();
-                        info!("schedule_once:interval id_{:016x} ch(prev: {}, exec: {}|{}), runnnable: {}, contended: {}, (immediate+provisional)/max: ({}+{})/{} uncontended: {} stuck: {} completed: {}!", random_id, from_prev.len(), to_execute_substage.len(), from_exec.len(), runnable_queue.task_count(), contended_count, executing_queue_count, provisioning_tracker_count, max_executing_queue_count, address_book.uncontended_task_ids.len(), address_book.stuck_tasks.len(), completed_count);
+                        info!("schedule_once:interval id_{:016x} ch(prev: {}, exec: {}|{}), runnnable: {}, contended: {}, (immediate+provisional)/max: ({}+{})/{} uncontended: {} stuck: {} processed: {}!", random_id, from_prev.len(), to_execute_substage.len(), from_exec.len(), runnable_queue.task_count(), contended_count, executing_queue_count, provisioning_tracker_count, max_executing_queue_count, address_book.uncontended_task_ids.len(), address_book.stuck_tasks.len(), processed_count);
                     }
                 }
                 while executing_queue_count + provisioning_tracker_count < max_executing_queue_count {
@@ -1549,10 +1549,10 @@ impl ScheduleStage {
                         executing_queue_count = executing_queue_count.checked_add(1).unwrap();
                         to_execute_substage.send(ExecutablePayload(ee)).unwrap();
                     }
-                    trace!("schedule_once id_{:016x} [R] ch(prev: {}, exec: {}|{}), runnnable: {}, contended: {}, (immediate+provisional)/max: ({}+{})/{} uncontended: {} stuck: {} completed: {}!", random_id, from_prev.len(), to_execute_substage.len(), from_exec.len(), runnable_queue.task_count(), contended_count, executing_queue_count, provisioning_tracker_count, max_executing_queue_count, address_book.uncontended_task_ids.len(), address_book.stuck_tasks.len(), completed_count);
+                    trace!("schedule_once id_{:016x} [R] ch(prev: {}, exec: {}|{}), runnnable: {}, contended: {}, (immediate+provisional)/max: ({}+{})/{} uncontended: {} stuck: {} processed: {}!", random_id, from_prev.len(), to_execute_substage.len(), from_exec.len(), runnable_queue.task_count(), contended_count, executing_queue_count, provisioning_tracker_count, max_executing_queue_count, address_book.uncontended_task_ids.len(), address_book.stuck_tasks.len(), processed_count);
                     if start.elapsed() > std::time::Duration::from_millis(150) {
                         start = std::time::Instant::now();
-                        info!("schedule_once:interval id_{:016x} ch(prev: {}, exec: {}|{}), runnnable: {}, contended: {}, (immediate+provisional)/max: ({}+{})/{} uncontended: {} stuck: {} completed: {}!", random_id, from_prev.len(), to_execute_substage.len(), from_exec.len(), runnable_queue.task_count(), contended_count, executing_queue_count, provisioning_tracker_count, max_executing_queue_count, address_book.uncontended_task_ids.len(), address_book.stuck_tasks.len(), completed_count);
+                        info!("schedule_once:interval id_{:016x} ch(prev: {}, exec: {}|{}), runnnable: {}, contended: {}, (immediate+provisional)/max: ({}+{})/{} uncontended: {} stuck: {} processed: {}!", random_id, from_prev.len(), to_execute_substage.len(), from_exec.len(), runnable_queue.task_count(), contended_count, executing_queue_count, provisioning_tracker_count, max_executing_queue_count, address_book.uncontended_task_ids.len(), address_book.stuck_tasks.len(), processed_count);
                     } else {
                         break;
                     }
@@ -1583,15 +1583,15 @@ impl ScheduleStage {
                         from_exec_len = from_exec_len.checked_sub(1).unwrap();
                         empty_from_exec = from_exec_len == 0;
                         executing_queue_count = executing_queue_count.checked_sub(1).unwrap();
-                        completed_count = completed_count.checked_add(1).unwrap();
-                        Self::commit_completed_execution(
+                        processed_count = processed_count.checked_add(1).unwrap();
+                        Self::commit_processed_execution(
                             ast,
                             &mut processed_execution_environment,
                             address_book,
                             &mut commit_clock,
                             &mut provisioning_tracker_count,
                         );
-                        to_next_stage.send(DroppablePayload(processed_execution_environment)).unwrap();
+                        to_next_stage.send(ExaminablePayload(processed_execution_environment)).unwrap();
                     }
                     if !empty_from {
                         let task = from_prev.recv().unwrap().0;
@@ -1614,7 +1614,7 @@ impl ScheduleStage {
             indexer_handle.join().unwrap().unwrap();
         }
 
-        info!("schedule_once:final id_{:016x} (from_disconnected: {}, from_exec_disconnected: {}, no_more_work: {}) ch(prev: {}, exec: {}|{}), runnnable: {}, contended: {}, (immediate+provisional)/max: ({}+{})/{} uncontended: {} stuck: {} completed: {}!", random_id, from_disconnected, from_exec_disconnected, no_more_work, from_prev.len(), to_execute_substage.len(), from_exec.len(), runnable_queue.task_count(), contended_count, executing_queue_count, provisioning_tracker_count, max_executing_queue_count, address_book.uncontended_task_ids.len(), address_book.stuck_tasks.len(), completed_count);
+        info!("schedule_once:final id_{:016x} (from_disconnected: {}, from_exec_disconnected: {}, no_more_work: {}) ch(prev: {}, exec: {}|{}), runnnable: {}, contended: {}, (immediate+provisional)/max: ({}+{})/{} uncontended: {} stuck: {} processed: {}!", random_id, from_disconnected, from_exec_disconnected, no_more_work, from_prev.len(), to_execute_substage.len(), from_exec.len(), runnable_queue.task_count(), contended_count, executing_queue_count, provisioning_tracker_count, max_executing_queue_count, address_book.uncontended_task_ids.len(), address_book.stuck_tasks.len(), processed_count);
     }
 
     pub fn run(
@@ -1624,7 +1624,7 @@ impl ScheduleStage {
         from: &crossbeam_channel::Receiver<SchedulablePayload>,
         to_execute_substage: &crossbeam_channel::Sender<ExecutablePayload>,
         from_execute_substage: &crossbeam_channel::Receiver<UnlockablePayload>,
-        maybe_to_next_stage: Option<&crossbeam_channel::Sender<DroppablePayload>>, // assume nonblocking
+        maybe_to_next_stage: Option<&crossbeam_channel::Sender<ExaminablePayload>>, // assume nonblocking
     ) {
         #[derive(Clone, Copy, Debug)]
         struct AtTopOfScheduleThread;
@@ -1647,7 +1647,7 @@ impl ScheduleStage {
 pub struct SchedulablePayload(pub TaskInQueue);
 pub struct ExecutablePayload(pub Box<ExecutionEnvironment>);
 pub struct UnlockablePayload(pub Box<ExecutionEnvironment>);
-pub struct DroppablePayload(pub Box<ExecutionEnvironment>);
+pub struct ExaminablePayload(pub Box<ExecutionEnvironment>);
 
 struct ExecuteStage {
     //bank: Bank,

From 813b949bb2fb11f140907763c3180f06d89da560 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 9 Sep 2022 15:24:57 +0900
Subject: [PATCH 2328/3199] save

---
 runtime/src/bank.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 67ec5db54d329b..c37713bd55eac9 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1323,7 +1323,7 @@ impl Default for Scheduler {
                         ("compute_units", ee.cu, i64),
                     );
                 }
-                completed_ee_sender.send(solana_scheduler::UnlockablePayload(ee)).unwrap();
+                processed_ee_sender.send(solana_scheduler::UnlockablePayload(ee)).unwrap();
             }
             Ok(())
         }).unwrap()}).collect();

From fd015c85121830e22f5c7c3f06bd06e089232b4d Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 9 Sep 2022 15:29:22 +0900
Subject: [PATCH 2329/3199] save

---
 runtime/src/bank.rs | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index c37713bd55eac9..8745fee8be4426 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1302,7 +1302,8 @@ impl Default for Scheduler {
                 } = tx_results;
 
                 ee.execution_result = Some(fee_collection_results.into_iter().collect::<Result<_>>());
-                ee.cu = execution_results[0].details().unwrap().executed_units;
+                let details = execution_results[0].details().unwrap();
+                ee.cu = details.executed_units;
 
 
                 //ee.reindex_with_address_book();
@@ -1318,7 +1319,7 @@ impl Default for Scheduler {
                         ("thread", current_thread_name, String),
                         ("signature", &sig, String),
                         ("account_locks_in_json", "{}", String),
-                        ("status", format!("{:?}", ee.execution_result.as_ref().unwrap()), String),
+                        ("status", format!("{:?}", details.status), String),
                         ("duration", duration_with_overhead, i64),
                         ("compute_units", ee.cu, i64),
                     );

From 512e401732318a27a249c9e8fcd0b59164b65c34 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 9 Sep 2022 15:31:11 +0900
Subject: [PATCH 2330/3199] save

---
 runtime/src/bank.rs | 1 +
 1 file changed, 1 insertion(+)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 8745fee8be4426..0337a2ae5b1112 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1243,6 +1243,7 @@ impl Default for Scheduler {
         let (processed_ee_sender, processed_ee_receiver) = crossbeam_channel::unbounded();
         let (retired_ee_sender, retired_ee_receiver) = crossbeam_channel::unbounded();
 
+        // todo: drop this arc correctly at wait_for_scheduler()....
         let bank = Arc::new(std::sync::RwLock::new(None::<Arc<Bank>>));
 
         let executing_thread_count = std::env::var("EXECUTING_THREAD_COUNT")

From 8d9b1070e9c56b489ee5b5eebcba5b44d5badf9f Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 9 Sep 2022 15:32:06 +0900
Subject: [PATCH 2331/3199] save

---
 runtime/src/bank.rs | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 0337a2ae5b1112..28c4c8b32db14e 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1273,8 +1273,7 @@ impl Default for Scheduler {
                 trace!("execute_substage: thread: {} transaction_index: {} execute_clock: {}", thx, ee.task.transaction_index_in_entries_for_replay(), current_execute_clock);
 
                 let bank_r = bank.read().unwrap();
-                let bank_o = (&bank_r).as_ref().unwrap();
-                let bank = bank_o.as_ref();
+                let bank = (&bank_r).as_ref().unwrap();
 
                 let tx_account_lock_limit = bank.get_transaction_account_lock_limit();
                 let lock_result = ee.task.tx.0

From 16a29976e65aa81ba9e90c47c2545b4cfd70a05b Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 9 Sep 2022 15:32:25 +0900
Subject: [PATCH 2332/3199] save

---
 runtime/src/bank.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 28c4c8b32db14e..d25dd92a74ecf4 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1273,7 +1273,7 @@ impl Default for Scheduler {
                 trace!("execute_substage: thread: {} transaction_index: {} execute_clock: {}", thx, ee.task.transaction_index_in_entries_for_replay(), current_execute_clock);
 
                 let bank_r = bank.read().unwrap();
-                let bank = (&bank_r).as_ref().unwrap();
+                let bank = bank_r.as_ref().unwrap();
 
                 let tx_account_lock_limit = bank.get_transaction_account_lock_limit();
                 let lock_result = ee.task.tx.0

From 5d259d5077890d8ea74f83f092ab7ae5bb86a038 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 9 Sep 2022 21:55:17 +0900
Subject: [PATCH 2333/3199] save

---
 scheduler/src/lib.rs | 34 ++++++++++++++++++++++------------
 1 file changed, 22 insertions(+), 12 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index dbd216e6a17b99..4fce1a9b780134 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1404,6 +1404,8 @@ impl ScheduleStage {
         let mut execute_clock = 0;
         let mut commit_clock = 0;
         let mut processed_count = 0_usize;
+        let mut interval_count = 0;
+
         assert!(max_executing_queue_count > 0);
 
         let (ee_sender, ee_receiver) = crossbeam_channel::unbounded::<ExaminablePayload>();
@@ -1460,7 +1462,7 @@ impl ScheduleStage {
                 })
                 .unwrap()
         }).collect::<Vec<_>>();
-        let mut start = std::time::Instant::now();
+        let mut (start, last_processed_count) = (std::time::Instant::now(), 0);
 
         let (mut from_disconnected, mut from_exec_disconnected, mut no_more_work): (bool, bool, bool) = Default::default();
         loop {
@@ -1524,10 +1526,15 @@ impl ScheduleStage {
                         executing_queue_count = executing_queue_count.checked_add(1).unwrap();
                         to_execute_substage.send(ExecutablePayload(ee)).unwrap();
                     }
-                    trace!("schedule_once id_{:016x} [C] ch(prev: {}, exec: {}|{}), runnnable: {}, contended: {}, (immediate+provisional)/max: ({}+{})/{} uncontended: {} stuck: {} processed: {}!", random_id, from_prev.len(), to_execute_substage.len(), from_exec.len(), runnable_queue.task_count(), contended_count, executing_queue_count, provisioning_tracker_count, max_executing_queue_count, address_book.uncontended_task_ids.len(), address_book.stuck_tasks.len(), processed_count);
-                    if start.elapsed() > std::time::Duration::from_millis(150) {
-                        start = std::time::Instant::now();
-                        info!("schedule_once:interval id_{:016x} ch(prev: {}, exec: {}|{}), runnnable: {}, contended: {}, (immediate+provisional)/max: ({}+{})/{} uncontended: {} stuck: {} processed: {}!", random_id, from_prev.len(), to_execute_substage.len(), from_exec.len(), runnable_queue.task_count(), contended_count, executing_queue_count, provisioning_tracker_count, max_executing_queue_count, address_book.uncontended_task_ids.len(), address_book.stuck_tasks.len(), processed_count);
+                    debug!("schedule_once id_{:016x} [C] ch(prev: {}, exec: {}|{}), r: {}, u/c: {}/{}, (imm+provi)/max: ({}+{})/{} s: {} done: {}", random_id, from_prev.len(), to_execute_substage.len(), from_exec.len(), runnable_queue.task_count(), address_book.uncontended_task_ids.len(), contended_count, executing_queue_count, provisioning_tracker_count, max_executing_queue_count, address_book.stuck_tasks.len(), processed_count);
+                    interval_count += 1;
+                    if interval_count % 100 {
+                        let elapsed = start.elapsed();
+                        if elapsed > std::time::Duration::from_millis(150) {
+                            let delta = processed_count - last_processed_count;
+                            info!("schedule_once:interval id_{:016x} ch(prev: {}, exec: {}|{}), r: {}, u/c: {}/{}, (imm+provi)/max: ({}+{})/{} s: {} done: {} ({}txs/{}us={}tps)", random_id, from_prev.len(), to_execute_substage.len(), from_exec.len(), runnable_queue.task_count(), address_book.uncontended_task_ids.len(), contended_count, executing_queue_count, provisioning_tracker_count, max_executing_queue_count, address_book.stuck_tasks.len(), processed_count, delta, elapsed.as_micros(), );
+                            (start, last_processed_count) = (std::time::Instant::now(), processed_count);
+                        }
                     }
                 }
                 while executing_queue_count + provisioning_tracker_count < max_executing_queue_count {
@@ -1549,15 +1556,18 @@ impl ScheduleStage {
                         executing_queue_count = executing_queue_count.checked_add(1).unwrap();
                         to_execute_substage.send(ExecutablePayload(ee)).unwrap();
                     }
-                    trace!("schedule_once id_{:016x} [R] ch(prev: {}, exec: {}|{}), runnnable: {}, contended: {}, (immediate+provisional)/max: ({}+{})/{} uncontended: {} stuck: {} processed: {}!", random_id, from_prev.len(), to_execute_substage.len(), from_exec.len(), runnable_queue.task_count(), contended_count, executing_queue_count, provisioning_tracker_count, max_executing_queue_count, address_book.uncontended_task_ids.len(), address_book.stuck_tasks.len(), processed_count);
-                    if start.elapsed() > std::time::Duration::from_millis(150) {
-                        start = std::time::Instant::now();
-                        info!("schedule_once:interval id_{:016x} ch(prev: {}, exec: {}|{}), runnnable: {}, contended: {}, (immediate+provisional)/max: ({}+{})/{} uncontended: {} stuck: {} processed: {}!", random_id, from_prev.len(), to_execute_substage.len(), from_exec.len(), runnable_queue.task_count(), contended_count, executing_queue_count, provisioning_tracker_count, max_executing_queue_count, address_book.uncontended_task_ids.len(), address_book.stuck_tasks.len(), processed_count);
-                    } else {
-                        break;
+                    debug!("schedule_once id_{:016x} [R] ch(prev: {}, exec: {}|{}), r: {}, u/c: {}/{}, (imm+provi)/max: ({}+{})/{} s: {} done: {}", random_id, from_prev.len(), to_execute_substage.len(), from_exec.len(), runnable_queue.task_count(), address_book.uncontended_task_ids.len(), contended_count, executing_queue_count, provisioning_tracker_count, max_executing_queue_count, address_book.stuck_tasks.len(), processed_count);
+                    interval_count += 1;
+                    if interval_count % 100 {
+                        let elapsed = start.elapsed();
+                        if elapsed > std::time::Duration::from_millis(150) {
+                            let delta = processed_count - last_processed_count;
+                            info!("schedule_once:interval id_{:016x} ch(prev: {}, exec: {}|{}), r: {}, u/c: {}/{}, (imm+provi)/max: ({}+{})/{} s: {} done: {} ({}txs/{}us={}tps)", random_id, from_prev.len(), to_execute_substage.len(), from_exec.len(), runnable_queue.task_count(), address_book.uncontended_task_ids.len(), contended_count, executing_queue_count, provisioning_tracker_count, max_executing_queue_count, address_book.stuck_tasks.len(), processed_count, delta, elapsed.as_micros(), );
+                            (start, last_processed_count) = (std::time::Instant::now(), processed_count);
+                        }
                     }
                     if !from_exec.is_empty() {
-                        trace!("abort aggressive readalbe queue processing due to non-empty from_exec");
+                        trace!("abort aggressive readable queue processing due to non-empty from_exec");
                         break;
                     }
                 }

From 08664443bfc95589b9afa0a8ff19f2cd41e97b6e Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 9 Sep 2022 21:56:47 +0900
Subject: [PATCH 2334/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 4fce1a9b780134..88b0760686667a 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1462,7 +1462,7 @@ impl ScheduleStage {
                 })
                 .unwrap()
         }).collect::<Vec<_>>();
-        let mut (start, last_processed_count) = (std::time::Instant::now(), 0);
+        let (mut start, mut last_processed_count) = (std::time::Instant::now(), 0);
 
         let (mut from_disconnected, mut from_exec_disconnected, mut no_more_work): (bool, bool, bool) = Default::default();
         loop {

From 75fbc3484c8e6ba9500af3137f5984a08114ce7a Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 9 Sep 2022 21:58:22 +0900
Subject: [PATCH 2335/3199] save

---
 scheduler/src/lib.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 88b0760686667a..f30d8092e7b210 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1526,7 +1526,7 @@ impl ScheduleStage {
                         executing_queue_count = executing_queue_count.checked_add(1).unwrap();
                         to_execute_substage.send(ExecutablePayload(ee)).unwrap();
                     }
-                    debug!("schedule_once id_{:016x} [C] ch(prev: {}, exec: {}|{}), r: {}, u/c: {}/{}, (imm+provi)/max: ({}+{})/{} s: {} done: {}", random_id, from_prev.len(), to_execute_substage.len(), from_exec.len(), runnable_queue.task_count(), address_book.uncontended_task_ids.len(), contended_count, executing_queue_count, provisioning_tracker_count, max_executing_queue_count, address_book.stuck_tasks.len(), processed_count);
+                    //debug!("schedule_once id_{:016x} [C] ch(prev: {}, exec: {}|{}), r: {}, u/c: {}/{}, (imm+provi)/max: ({}+{})/{} s: {} done: {}", random_id, from_prev.len(), to_execute_substage.len(), from_exec.len(), runnable_queue.task_count(), address_book.uncontended_task_ids.len(), contended_count, executing_queue_count, provisioning_tracker_count, max_executing_queue_count, address_book.stuck_tasks.len(), processed_count);
                     interval_count += 1;
                     if interval_count % 100 {
                         let elapsed = start.elapsed();
@@ -1556,7 +1556,7 @@ impl ScheduleStage {
                         executing_queue_count = executing_queue_count.checked_add(1).unwrap();
                         to_execute_substage.send(ExecutablePayload(ee)).unwrap();
                     }
-                    debug!("schedule_once id_{:016x} [R] ch(prev: {}, exec: {}|{}), r: {}, u/c: {}/{}, (imm+provi)/max: ({}+{})/{} s: {} done: {}", random_id, from_prev.len(), to_execute_substage.len(), from_exec.len(), runnable_queue.task_count(), address_book.uncontended_task_ids.len(), contended_count, executing_queue_count, provisioning_tracker_count, max_executing_queue_count, address_book.stuck_tasks.len(), processed_count);
+                    //debug!("schedule_once id_{:016x} [R] ch(prev: {}, exec: {}|{}), r: {}, u/c: {}/{}, (imm+provi)/max: ({}+{})/{} s: {} done: {}", random_id, from_prev.len(), to_execute_substage.len(), from_exec.len(), runnable_queue.task_count(), address_book.uncontended_task_ids.len(), contended_count, executing_queue_count, provisioning_tracker_count, max_executing_queue_count, address_book.stuck_tasks.len(), processed_count);
                     interval_count += 1;
                     if interval_count % 100 {
                         let elapsed = start.elapsed();

From fa880b3a11ea7d242695f0279592a3ec4ee4f1c4 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 9 Sep 2022 21:59:46 +0900
Subject: [PATCH 2336/3199] save

---
 scheduler/src/lib.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index f30d8092e7b210..f9293702f3b3ef 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1532,7 +1532,7 @@ impl ScheduleStage {
                         let elapsed = start.elapsed();
                         if elapsed > std::time::Duration::from_millis(150) {
                             let delta = processed_count - last_processed_count;
-                            info!("schedule_once:interval id_{:016x} ch(prev: {}, exec: {}|{}), r: {}, u/c: {}/{}, (imm+provi)/max: ({}+{})/{} s: {} done: {} ({}txs/{}us={}tps)", random_id, from_prev.len(), to_execute_substage.len(), from_exec.len(), runnable_queue.task_count(), address_book.uncontended_task_ids.len(), contended_count, executing_queue_count, provisioning_tracker_count, max_executing_queue_count, address_book.stuck_tasks.len(), processed_count, delta, elapsed.as_micros(), );
+                            info!("schedule_once:interval id_{:016x} ch(prev: {}, exec: {}|{}), r: {}, u/c: {}/{}, (imm+provi)/max: ({}+{})/{} s: {} done: {} ({}txs/{}us={}tps)", random_id, from_prev.len(), to_execute_substage.len(), from_exec.len(), runnable_queue.task_count(), address_book.uncontended_task_ids.len(), contended_count, executing_queue_count, provisioning_tracker_count, max_executing_queue_count, address_book.stuck_tasks.len(), processed_count, delta, elapsed.as_micros(), 1000*delta/elapsed);
                             (start, last_processed_count) = (std::time::Instant::now(), processed_count);
                         }
                     }
@@ -1562,7 +1562,7 @@ impl ScheduleStage {
                         let elapsed = start.elapsed();
                         if elapsed > std::time::Duration::from_millis(150) {
                             let delta = processed_count - last_processed_count;
-                            info!("schedule_once:interval id_{:016x} ch(prev: {}, exec: {}|{}), r: {}, u/c: {}/{}, (imm+provi)/max: ({}+{})/{} s: {} done: {} ({}txs/{}us={}tps)", random_id, from_prev.len(), to_execute_substage.len(), from_exec.len(), runnable_queue.task_count(), address_book.uncontended_task_ids.len(), contended_count, executing_queue_count, provisioning_tracker_count, max_executing_queue_count, address_book.stuck_tasks.len(), processed_count, delta, elapsed.as_micros(), );
+                            info!("schedule_once:interval id_{:016x} ch(prev: {}, exec: {}|{}), r: {}, u/c: {}/{}, (imm+provi)/max: ({}+{})/{} s: {} done: {} ({}txs/{}us={}tps)", random_id, from_prev.len(), to_execute_substage.len(), from_exec.len(), runnable_queue.task_count(), address_book.uncontended_task_ids.len(), contended_count, executing_queue_count, provisioning_tracker_count, max_executing_queue_count, address_book.stuck_tasks.len(), processed_count, delta, elapsed.as_micros(), 1000*delta/elapsed);
                             (start, last_processed_count) = (std::time::Instant::now(), processed_count);
                         }
                     }

From e49c9e1e6b371aa05247145e0bf2206daefd5a9f Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 9 Sep 2022 22:00:00 +0900
Subject: [PATCH 2337/3199] save

---
 scheduler/src/lib.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index f9293702f3b3ef..a7dc1145c796f1 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1528,7 +1528,7 @@ impl ScheduleStage {
                     }
                     //debug!("schedule_once id_{:016x} [C] ch(prev: {}, exec: {}|{}), r: {}, u/c: {}/{}, (imm+provi)/max: ({}+{})/{} s: {} done: {}", random_id, from_prev.len(), to_execute_substage.len(), from_exec.len(), runnable_queue.task_count(), address_book.uncontended_task_ids.len(), contended_count, executing_queue_count, provisioning_tracker_count, max_executing_queue_count, address_book.stuck_tasks.len(), processed_count);
                     interval_count += 1;
-                    if interval_count % 100 {
+                    if interval_count % 100 == 0 {
                         let elapsed = start.elapsed();
                         if elapsed > std::time::Duration::from_millis(150) {
                             let delta = processed_count - last_processed_count;
@@ -1558,7 +1558,7 @@ impl ScheduleStage {
                     }
                     //debug!("schedule_once id_{:016x} [R] ch(prev: {}, exec: {}|{}), r: {}, u/c: {}/{}, (imm+provi)/max: ({}+{})/{} s: {} done: {}", random_id, from_prev.len(), to_execute_substage.len(), from_exec.len(), runnable_queue.task_count(), address_book.uncontended_task_ids.len(), contended_count, executing_queue_count, provisioning_tracker_count, max_executing_queue_count, address_book.stuck_tasks.len(), processed_count);
                     interval_count += 1;
-                    if interval_count % 100 {
+                    if interval_count % 100 == 0 {
                         let elapsed = start.elapsed();
                         if elapsed > std::time::Duration::from_millis(150) {
                             let delta = processed_count - last_processed_count;

From b9a5ece98e1140d0173569ea2f0d001c87a30077 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 9 Sep 2022 22:00:29 +0900
Subject: [PATCH 2338/3199] save

---
 scheduler/src/lib.rs | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index a7dc1145c796f1..f8624d643218b1 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1532,6 +1532,7 @@ impl ScheduleStage {
                         let elapsed = start.elapsed();
                         if elapsed > std::time::Duration::from_millis(150) {
                             let delta = processed_count - last_processed_count;
+                            let elapsed = elapsed.as_micros();
                             info!("schedule_once:interval id_{:016x} ch(prev: {}, exec: {}|{}), r: {}, u/c: {}/{}, (imm+provi)/max: ({}+{})/{} s: {} done: {} ({}txs/{}us={}tps)", random_id, from_prev.len(), to_execute_substage.len(), from_exec.len(), runnable_queue.task_count(), address_book.uncontended_task_ids.len(), contended_count, executing_queue_count, provisioning_tracker_count, max_executing_queue_count, address_book.stuck_tasks.len(), processed_count, delta, elapsed.as_micros(), 1000*delta/elapsed);
                             (start, last_processed_count) = (std::time::Instant::now(), processed_count);
                         }
@@ -1562,6 +1563,7 @@ impl ScheduleStage {
                         let elapsed = start.elapsed();
                         if elapsed > std::time::Duration::from_millis(150) {
                             let delta = processed_count - last_processed_count;
+                            let elapsed = elapsed.as_micros();
                             info!("schedule_once:interval id_{:016x} ch(prev: {}, exec: {}|{}), r: {}, u/c: {}/{}, (imm+provi)/max: ({}+{})/{} s: {} done: {} ({}txs/{}us={}tps)", random_id, from_prev.len(), to_execute_substage.len(), from_exec.len(), runnable_queue.task_count(), address_book.uncontended_task_ids.len(), contended_count, executing_queue_count, provisioning_tracker_count, max_executing_queue_count, address_book.stuck_tasks.len(), processed_count, delta, elapsed.as_micros(), 1000*delta/elapsed);
                             (start, last_processed_count) = (std::time::Instant::now(), processed_count);
                         }

From e03a12f7d625e1a1fcfdc0423253a993c0f65006 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 9 Sep 2022 22:01:13 +0900
Subject: [PATCH 2339/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index f8624d643218b1..7b47db94c9e4d3 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1560,7 +1560,7 @@ impl ScheduleStage {
                     //debug!("schedule_once id_{:016x} [R] ch(prev: {}, exec: {}|{}), r: {}, u/c: {}/{}, (imm+provi)/max: ({}+{})/{} s: {} done: {}", random_id, from_prev.len(), to_execute_substage.len(), from_exec.len(), runnable_queue.task_count(), address_book.uncontended_task_ids.len(), contended_count, executing_queue_count, provisioning_tracker_count, max_executing_queue_count, address_book.stuck_tasks.len(), processed_count);
                     interval_count += 1;
                     if interval_count % 100 == 0 {
-                        let elapsed = start.elapsed();
+                        let elapsed = start.since();
                         if elapsed > std::time::Duration::from_millis(150) {
                             let delta = processed_count - last_processed_count;
                             let elapsed = elapsed.as_micros();

From 0106e8137a9900f1af15763bd9fbed326fce253e Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 9 Sep 2022 22:01:44 +0900
Subject: [PATCH 2340/3199] save

---
 scheduler/src/lib.rs | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 7b47db94c9e4d3..4dd2b9ace983e1 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1560,11 +1560,11 @@ impl ScheduleStage {
                     //debug!("schedule_once id_{:016x} [R] ch(prev: {}, exec: {}|{}), r: {}, u/c: {}/{}, (imm+provi)/max: ({}+{})/{} s: {} done: {}", random_id, from_prev.len(), to_execute_substage.len(), from_exec.len(), runnable_queue.task_count(), address_book.uncontended_task_ids.len(), contended_count, executing_queue_count, provisioning_tracker_count, max_executing_queue_count, address_book.stuck_tasks.len(), processed_count);
                     interval_count += 1;
                     if interval_count % 100 == 0 {
-                        let elapsed = start.since();
+                        let elapsed = start.elapsed();
                         if elapsed > std::time::Duration::from_millis(150) {
                             let delta = processed_count - last_processed_count;
-                            let elapsed = elapsed.as_micros();
-                            info!("schedule_once:interval id_{:016x} ch(prev: {}, exec: {}|{}), r: {}, u/c: {}/{}, (imm+provi)/max: ({}+{})/{} s: {} done: {} ({}txs/{}us={}tps)", random_id, from_prev.len(), to_execute_substage.len(), from_exec.len(), runnable_queue.task_count(), address_book.uncontended_task_ids.len(), contended_count, executing_queue_count, provisioning_tracker_count, max_executing_queue_count, address_book.stuck_tasks.len(), processed_count, delta, elapsed.as_micros(), 1000*delta/elapsed);
+                            let elapsed2 = elapsed.as_micros();
+                            info!("schedule_once:interval id_{:016x} ch(prev: {}, exec: {}|{}), r: {}, u/c: {}/{}, (imm+provi)/max: ({}+{})/{} s: {} done: {} ({}txs/{}us={}tps)", random_id, from_prev.len(), to_execute_substage.len(), from_exec.len(), runnable_queue.task_count(), address_book.uncontended_task_ids.len(), contended_count, executing_queue_count, provisioning_tracker_count, max_executing_queue_count, address_book.stuck_tasks.len(), processed_count, delta, elapsed.as_micros(), 1000*delta/elapsed2);
                             (start, last_processed_count) = (std::time::Instant::now(), processed_count);
                         }
                     }

From 9c3139f34a6c17464c78286ac050c59e53995cb2 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 9 Sep 2022 22:02:03 +0900
Subject: [PATCH 2341/3199] save

---
 scheduler/src/lib.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 4dd2b9ace983e1..6e6a31ca2bcac3 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1532,8 +1532,8 @@ impl ScheduleStage {
                         let elapsed = start.elapsed();
                         if elapsed > std::time::Duration::from_millis(150) {
                             let delta = processed_count - last_processed_count;
-                            let elapsed = elapsed.as_micros();
-                            info!("schedule_once:interval id_{:016x} ch(prev: {}, exec: {}|{}), r: {}, u/c: {}/{}, (imm+provi)/max: ({}+{})/{} s: {} done: {} ({}txs/{}us={}tps)", random_id, from_prev.len(), to_execute_substage.len(), from_exec.len(), runnable_queue.task_count(), address_book.uncontended_task_ids.len(), contended_count, executing_queue_count, provisioning_tracker_count, max_executing_queue_count, address_book.stuck_tasks.len(), processed_count, delta, elapsed.as_micros(), 1000*delta/elapsed);
+                            let elapsed2 = elapsed.as_micros();
+                            info!("schedule_once:interval id_{:016x} ch(prev: {}, exec: {}|{}), r: {}, u/c: {}/{}, (imm+provi)/max: ({}+{})/{} s: {} done: {} ({}txs/{}us={}tps)", random_id, from_prev.len(), to_execute_substage.len(), from_exec.len(), runnable_queue.task_count(), address_book.uncontended_task_ids.len(), contended_count, executing_queue_count, provisioning_tracker_count, max_executing_queue_count, address_book.stuck_tasks.len(), processed_count, delta, elapsed.as_micros(), 1000*delta/elapsed2);
                             (start, last_processed_count) = (std::time::Instant::now(), processed_count);
                         }
                     }

From 9d23a7eb381f1145fa5095d655e7849ee1f33972 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 9 Sep 2022 22:02:30 +0900
Subject: [PATCH 2342/3199] save

---
 scheduler/src/lib.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 6e6a31ca2bcac3..6c9041968b2656 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1533,7 +1533,7 @@ impl ScheduleStage {
                         if elapsed > std::time::Duration::from_millis(150) {
                             let delta = processed_count - last_processed_count;
                             let elapsed2 = elapsed.as_micros();
-                            info!("schedule_once:interval id_{:016x} ch(prev: {}, exec: {}|{}), r: {}, u/c: {}/{}, (imm+provi)/max: ({}+{})/{} s: {} done: {} ({}txs/{}us={}tps)", random_id, from_prev.len(), to_execute_substage.len(), from_exec.len(), runnable_queue.task_count(), address_book.uncontended_task_ids.len(), contended_count, executing_queue_count, provisioning_tracker_count, max_executing_queue_count, address_book.stuck_tasks.len(), processed_count, delta, elapsed.as_micros(), 1000*delta/elapsed2);
+                            info!("schedule_once:interval id_{:016x} ch(prev: {}, exec: {}|{}), r: {}, u/c: {}/{}, (imm+provi)/max: ({}+{})/{} s: {} done: {} ({}txs/{}us={}tps)", random_id, from_prev.len(), to_execute_substage.len(), from_exec.len(), runnable_queue.task_count(), address_book.uncontended_task_ids.len(), contended_count, executing_queue_count, provisioning_tracker_count, max_executing_queue_count, address_book.stuck_tasks.len(), processed_count, delta, elapsed.as_micros(), 1000_u128*delta/elapsed2);
                             (start, last_processed_count) = (std::time::Instant::now(), processed_count);
                         }
                     }
@@ -1564,7 +1564,7 @@ impl ScheduleStage {
                         if elapsed > std::time::Duration::from_millis(150) {
                             let delta = processed_count - last_processed_count;
                             let elapsed2 = elapsed.as_micros();
-                            info!("schedule_once:interval id_{:016x} ch(prev: {}, exec: {}|{}), r: {}, u/c: {}/{}, (imm+provi)/max: ({}+{})/{} s: {} done: {} ({}txs/{}us={}tps)", random_id, from_prev.len(), to_execute_substage.len(), from_exec.len(), runnable_queue.task_count(), address_book.uncontended_task_ids.len(), contended_count, executing_queue_count, provisioning_tracker_count, max_executing_queue_count, address_book.stuck_tasks.len(), processed_count, delta, elapsed.as_micros(), 1000*delta/elapsed2);
+                            info!("schedule_once:interval id_{:016x} ch(prev: {}, exec: {}|{}), r: {}, u/c: {}/{}, (imm+provi)/max: ({}+{})/{} s: {} done: {} ({}txs/{}us={}tps)", random_id, from_prev.len(), to_execute_substage.len(), from_exec.len(), runnable_queue.task_count(), address_book.uncontended_task_ids.len(), contended_count, executing_queue_count, provisioning_tracker_count, max_executing_queue_count, address_book.stuck_tasks.len(), processed_count, delta, elapsed.as_micros(), 1000_u128*delta/elapsed2);
                             (start, last_processed_count) = (std::time::Instant::now(), processed_count);
                         }
                     }

From 2954c47bbdd11aa469f03c5f7b505cd4286f82a6 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 9 Sep 2022 22:02:54 +0900
Subject: [PATCH 2343/3199] save

---
 scheduler/src/lib.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 6c9041968b2656..d1a65988c48433 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1531,7 +1531,7 @@ impl ScheduleStage {
                     if interval_count % 100 == 0 {
                         let elapsed = start.elapsed();
                         if elapsed > std::time::Duration::from_millis(150) {
-                            let delta = processed_count - last_processed_count;
+                            let delta = processed_count as u128 - last_processed_count;
                             let elapsed2 = elapsed.as_micros();
                             info!("schedule_once:interval id_{:016x} ch(prev: {}, exec: {}|{}), r: {}, u/c: {}/{}, (imm+provi)/max: ({}+{})/{} s: {} done: {} ({}txs/{}us={}tps)", random_id, from_prev.len(), to_execute_substage.len(), from_exec.len(), runnable_queue.task_count(), address_book.uncontended_task_ids.len(), contended_count, executing_queue_count, provisioning_tracker_count, max_executing_queue_count, address_book.stuck_tasks.len(), processed_count, delta, elapsed.as_micros(), 1000_u128*delta/elapsed2);
                             (start, last_processed_count) = (std::time::Instant::now(), processed_count);
@@ -1562,7 +1562,7 @@ impl ScheduleStage {
                     if interval_count % 100 == 0 {
                         let elapsed = start.elapsed();
                         if elapsed > std::time::Duration::from_millis(150) {
-                            let delta = processed_count - last_processed_count;
+                            let delta = processed_count as u128 - last_processed_count;
                             let elapsed2 = elapsed.as_micros();
                             info!("schedule_once:interval id_{:016x} ch(prev: {}, exec: {}|{}), r: {}, u/c: {}/{}, (imm+provi)/max: ({}+{})/{} s: {} done: {} ({}txs/{}us={}tps)", random_id, from_prev.len(), to_execute_substage.len(), from_exec.len(), runnable_queue.task_count(), address_book.uncontended_task_ids.len(), contended_count, executing_queue_count, provisioning_tracker_count, max_executing_queue_count, address_book.stuck_tasks.len(), processed_count, delta, elapsed.as_micros(), 1000_u128*delta/elapsed2);
                             (start, last_processed_count) = (std::time::Instant::now(), processed_count);

From c51bf9289271a8b0106652b8cbb07a1bf236bdca Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 9 Sep 2022 22:03:39 +0900
Subject: [PATCH 2344/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index d1a65988c48433..f6a6e80c1c174b 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1462,7 +1462,7 @@ impl ScheduleStage {
                 })
                 .unwrap()
         }).collect::<Vec<_>>();
-        let (mut start, mut last_processed_count) = (std::time::Instant::now(), 0);
+        let (mut start, mut last_processed_count) = (std::time::Instant::now(), 0_usize);
 
         let (mut from_disconnected, mut from_exec_disconnected, mut no_more_work): (bool, bool, bool) = Default::default();
         loop {

From f0ddf8e072cd3e0ade81f33ce940bbec5067c330 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 9 Sep 2022 22:04:17 +0900
Subject: [PATCH 2345/3199] save

---
 scheduler/src/lib.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index f6a6e80c1c174b..7cfd0c8594d10e 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1531,7 +1531,7 @@ impl ScheduleStage {
                     if interval_count % 100 == 0 {
                         let elapsed = start.elapsed();
                         if elapsed > std::time::Duration::from_millis(150) {
-                            let delta = processed_count as u128 - last_processed_count;
+                            let delta = (processed_count - last_processed_count) as u128;
                             let elapsed2 = elapsed.as_micros();
                             info!("schedule_once:interval id_{:016x} ch(prev: {}, exec: {}|{}), r: {}, u/c: {}/{}, (imm+provi)/max: ({}+{})/{} s: {} done: {} ({}txs/{}us={}tps)", random_id, from_prev.len(), to_execute_substage.len(), from_exec.len(), runnable_queue.task_count(), address_book.uncontended_task_ids.len(), contended_count, executing_queue_count, provisioning_tracker_count, max_executing_queue_count, address_book.stuck_tasks.len(), processed_count, delta, elapsed.as_micros(), 1000_u128*delta/elapsed2);
                             (start, last_processed_count) = (std::time::Instant::now(), processed_count);
@@ -1562,7 +1562,7 @@ impl ScheduleStage {
                     if interval_count % 100 == 0 {
                         let elapsed = start.elapsed();
                         if elapsed > std::time::Duration::from_millis(150) {
-                            let delta = processed_count as u128 - last_processed_count;
+                            let delta = (processed_count - last_processed_count) as u128;
                             let elapsed2 = elapsed.as_micros();
                             info!("schedule_once:interval id_{:016x} ch(prev: {}, exec: {}|{}), r: {}, u/c: {}/{}, (imm+provi)/max: ({}+{})/{} s: {} done: {} ({}txs/{}us={}tps)", random_id, from_prev.len(), to_execute_substage.len(), from_exec.len(), runnable_queue.task_count(), address_book.uncontended_task_ids.len(), contended_count, executing_queue_count, provisioning_tracker_count, max_executing_queue_count, address_book.stuck_tasks.len(), processed_count, delta, elapsed.as_micros(), 1000_u128*delta/elapsed2);
                             (start, last_processed_count) = (std::time::Instant::now(), processed_count);

From 668181b4cf758d76e9cd6a1f33db432395550f68 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 9 Sep 2022 22:05:19 +0900
Subject: [PATCH 2346/3199] save

---
 runtime/src/bank.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index d25dd92a74ecf4..1f2d252dd5feb8 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -6449,7 +6449,7 @@ impl Bank {
             if r.bank.read().unwrap().is_none() {
                 drop(r);
                 let w = self.scheduler.write().unwrap();
-                *w.bank.write().unwrap() = Some(Arc::clone(bank));
+                *w.bank.write().unwrap() = Some(Arc::clone(bank).downgrade());
                 drop(w);
                 self.scheduler.read().unwrap()
             } else {

From aa1bd26ea62a5b3a67ea8da3087e8993113973c1 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 9 Sep 2022 22:05:32 +0900
Subject: [PATCH 2347/3199] save

---
 runtime/src/bank.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 1f2d252dd5feb8..bc0f71c1a09a0c 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -6449,7 +6449,7 @@ impl Bank {
             if r.bank.read().unwrap().is_none() {
                 drop(r);
                 let w = self.scheduler.write().unwrap();
-                *w.bank.write().unwrap() = Some(Arc::clone(bank).downgrade());
+                *w.bank.write().unwrap() = Some(Arc::downgrade(Arc::clone(bank)));
                 drop(w);
                 self.scheduler.read().unwrap()
             } else {

From 637ef533fe9b0ba7e7e62e7c4e6246b15b559718 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 9 Sep 2022 22:05:49 +0900
Subject: [PATCH 2348/3199] save

---
 runtime/src/bank.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index bc0f71c1a09a0c..92fa5cafdff5fe 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -6449,7 +6449,7 @@ impl Bank {
             if r.bank.read().unwrap().is_none() {
                 drop(r);
                 let w = self.scheduler.write().unwrap();
-                *w.bank.write().unwrap() = Some(Arc::downgrade(Arc::clone(bank)));
+                *w.bank.write().unwrap() = Some(Arc::downgrade(&bank));
                 drop(w);
                 self.scheduler.read().unwrap()
             } else {

From 0be81e5c3da6e67484ccd0e1682e1c2b9bc9b1fb Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 9 Sep 2022 22:06:13 +0900
Subject: [PATCH 2349/3199] save

---
 runtime/src/bank.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 92fa5cafdff5fe..30298dee5ea8d8 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1205,7 +1205,7 @@ struct Scheduler {
     preloader: Arc<solana_scheduler::Preloader>,
     graceful_stop_initiated: bool,
     errors: Arc<std::sync::Mutex<Vec<Result<()>>>>,
-    bank: std::sync::Arc<std::sync::RwLock<std::option::Option<std::sync::Arc<Bank>>>>,
+    bank: std::sync::Arc<std::sync::RwLock<std::option::Option<std::sync::Weak<Bank>>>>,
     transaction_index: AtomicUsize,
 }
 

From c07af95dcfbd9ccb76bd43368a9729f260e14ec9 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 9 Sep 2022 22:06:38 +0900
Subject: [PATCH 2350/3199] save

---
 runtime/src/bank.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 30298dee5ea8d8..7d89cef05b1d16 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1244,7 +1244,7 @@ impl Default for Scheduler {
         let (retired_ee_sender, retired_ee_receiver) = crossbeam_channel::unbounded();
 
         // todo: drop this arc correctly at wait_for_scheduler()....
-        let bank = Arc::new(std::sync::RwLock::new(None::<Arc<Bank>>));
+        let bank = Arc::new(std::sync::RwLock::new(None::<Weak<Bank>>));
 
         let executing_thread_count = std::env::var("EXECUTING_THREAD_COUNT")
             .unwrap_or(format!("{}", 1))

From 6235dad9128cc104e080b415ad3e1bd3dc708504 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 9 Sep 2022 22:06:48 +0900
Subject: [PATCH 2351/3199] save

---
 runtime/src/bank.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 7d89cef05b1d16..8e765cf5992fe4 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1244,7 +1244,7 @@ impl Default for Scheduler {
         let (retired_ee_sender, retired_ee_receiver) = crossbeam_channel::unbounded();
 
         // todo: drop this arc correctly at wait_for_scheduler()....
-        let bank = Arc::new(std::sync::RwLock::new(None::<Weak<Bank>>));
+        let bank = Arc::new(std::sync::RwLock::new(None::<std::sync::Weak<Bank>>));
 
         let executing_thread_count = std::env::var("EXECUTING_THREAD_COUNT")
             .unwrap_or(format!("{}", 1))

From 3f5bccb856faa1b4cbb837d4d1e9008439ff6212 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 9 Sep 2022 22:07:19 +0900
Subject: [PATCH 2352/3199] save

---
 runtime/src/bank.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 8e765cf5992fe4..4356672332f24b 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1273,7 +1273,7 @@ impl Default for Scheduler {
                 trace!("execute_substage: thread: {} transaction_index: {} execute_clock: {}", thx, ee.task.transaction_index_in_entries_for_replay(), current_execute_clock);
 
                 let bank_r = bank.read().unwrap();
-                let bank = bank_r.as_ref().unwrap();
+                let bank = bank_r.as_ref().unwrap().unwrap();
 
                 let tx_account_lock_limit = bank.get_transaction_account_lock_limit();
                 let lock_result = ee.task.tx.0

From 87feb6b1325a9818ed08b29c578d8b4870fa7a26 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 9 Sep 2022 22:08:01 +0900
Subject: [PATCH 2353/3199] save

---
 runtime/src/bank.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 4356672332f24b..eb750678796de5 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1273,7 +1273,7 @@ impl Default for Scheduler {
                 trace!("execute_substage: thread: {} transaction_index: {} execute_clock: {}", thx, ee.task.transaction_index_in_entries_for_replay(), current_execute_clock);
 
                 let bank_r = bank.read().unwrap();
-                let bank = bank_r.as_ref().unwrap().unwrap();
+                let bank = bank_r.as_ref().unwrap().upgrade().unwrap();
 
                 let tx_account_lock_limit = bank.get_transaction_account_lock_limit();
                 let lock_result = ee.task.tx.0

From 9973c7cb73823bf9c89812fc64eeeeffbfe2e9ff Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 9 Sep 2022 22:08:23 +0900
Subject: [PATCH 2354/3199] save

---
 runtime/src/bank.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index eb750678796de5..dffe48de0b21ed 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1273,7 +1273,7 @@ impl Default for Scheduler {
                 trace!("execute_substage: thread: {} transaction_index: {} execute_clock: {}", thx, ee.task.transaction_index_in_entries_for_replay(), current_execute_clock);
 
                 let bank_r = bank.read().unwrap();
-                let bank = bank_r.as_ref().unwrap().upgrade().unwrap();
+                let bank = bank_r.as_ref().unwrap().upgrade().as_ref().unwrap();
 
                 let tx_account_lock_limit = bank.get_transaction_account_lock_limit();
                 let lock_result = ee.task.tx.0

From 690b5884f7801b8ffb7016ca5aa2f2624c78ab41 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 9 Sep 2022 22:08:55 +0900
Subject: [PATCH 2355/3199] save

---
 runtime/src/bank.rs | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index dffe48de0b21ed..d872604737ba52 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1273,7 +1273,8 @@ impl Default for Scheduler {
                 trace!("execute_substage: thread: {} transaction_index: {} execute_clock: {}", thx, ee.task.transaction_index_in_entries_for_replay(), current_execute_clock);
 
                 let bank_r = bank.read().unwrap();
-                let bank = bank_r.as_ref().unwrap().upgrade().as_ref().unwrap();
+                let bank_r2 = bank_r.as_ref().unwrap().upgrade();
+                let bank = bank_r2.as_ref().unwrap();
 
                 let tx_account_lock_limit = bank.get_transaction_account_lock_limit();
                 let lock_result = ee.task.tx.0

From bb265cfd5809d28eaaf1b7b8cb2fbc2a28413a01 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 9 Sep 2022 22:09:23 +0900
Subject: [PATCH 2356/3199] save

---
 runtime/src/bank.rs | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index d872604737ba52..0eb5790111521c 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1272,9 +1272,9 @@ impl Default for Scheduler {
                 execute_time += 1;
                 trace!("execute_substage: thread: {} transaction_index: {} execute_clock: {}", thx, ee.task.transaction_index_in_entries_for_replay(), current_execute_clock);
 
-                let bank_r = bank.read().unwrap();
-                let bank_r2 = bank_r.as_ref().unwrap().upgrade();
-                let bank = bank_r2.as_ref().unwrap();
+                let ro_bank = bank.read().unwrap();
+                let weak_bank = ro_bank.as_ref().unwrap().upgrade();
+                let bank = weak_bank.as_ref().unwrap();
 
                 let tx_account_lock_limit = bank.get_transaction_account_lock_limit();
                 let lock_result = ee.task.tx.0

From 97409097d89df96606b7e1de93fa9b0ee0503a0a Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 9 Sep 2022 22:12:56 +0900
Subject: [PATCH 2357/3199] save

---
 runtime/src/bank.rs  | 1 -
 scheduler/src/lib.rs | 4 ++--
 2 files changed, 2 insertions(+), 3 deletions(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 0eb5790111521c..da2e9ed0a10f21 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1243,7 +1243,6 @@ impl Default for Scheduler {
         let (processed_ee_sender, processed_ee_receiver) = crossbeam_channel::unbounded();
         let (retired_ee_sender, retired_ee_receiver) = crossbeam_channel::unbounded();
 
-        // todo: drop this arc correctly at wait_for_scheduler()....
         let bank = Arc::new(std::sync::RwLock::new(None::<std::sync::Weak<Bank>>));
 
         let executing_thread_count = std::env::var("EXECUTING_THREAD_COUNT")
diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 7cfd0c8594d10e..aaecf908b2d8e8 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1526,7 +1526,7 @@ impl ScheduleStage {
                         executing_queue_count = executing_queue_count.checked_add(1).unwrap();
                         to_execute_substage.send(ExecutablePayload(ee)).unwrap();
                     }
-                    //debug!("schedule_once id_{:016x} [C] ch(prev: {}, exec: {}|{}), r: {}, u/c: {}/{}, (imm+provi)/max: ({}+{})/{} s: {} done: {}", random_id, from_prev.len(), to_execute_substage.len(), from_exec.len(), runnable_queue.task_count(), address_book.uncontended_task_ids.len(), contended_count, executing_queue_count, provisioning_tracker_count, max_executing_queue_count, address_book.stuck_tasks.len(), processed_count);
+                    debug!("schedule_once id_{:016x} [C] ch(prev: {}, exec: {}|{}), r: {}, u/c: {}/{}, (imm+provi)/max: ({}+{})/{} s: {} done: {}", random_id, from_prev.len(), to_execute_substage.len(), from_exec.len(), runnable_queue.task_count(), address_book.uncontended_task_ids.len(), contended_count, executing_queue_count, provisioning_tracker_count, max_executing_queue_count, address_book.stuck_tasks.len(), processed_count);
                     interval_count += 1;
                     if interval_count % 100 == 0 {
                         let elapsed = start.elapsed();
@@ -1557,7 +1557,7 @@ impl ScheduleStage {
                         executing_queue_count = executing_queue_count.checked_add(1).unwrap();
                         to_execute_substage.send(ExecutablePayload(ee)).unwrap();
                     }
-                    //debug!("schedule_once id_{:016x} [R] ch(prev: {}, exec: {}|{}), r: {}, u/c: {}/{}, (imm+provi)/max: ({}+{})/{} s: {} done: {}", random_id, from_prev.len(), to_execute_substage.len(), from_exec.len(), runnable_queue.task_count(), address_book.uncontended_task_ids.len(), contended_count, executing_queue_count, provisioning_tracker_count, max_executing_queue_count, address_book.stuck_tasks.len(), processed_count);
+                    debug!("schedule_once id_{:016x} [R] ch(prev: {}, exec: {}|{}), r: {}, u/c: {}/{}, (imm+provi)/max: ({}+{})/{} s: {} done: {}", random_id, from_prev.len(), to_execute_substage.len(), from_exec.len(), runnable_queue.task_count(), address_book.uncontended_task_ids.len(), contended_count, executing_queue_count, provisioning_tracker_count, max_executing_queue_count, address_book.stuck_tasks.len(), processed_count);
                     interval_count += 1;
                     if interval_count % 100 == 0 {
                         let elapsed = start.elapsed();

From c26f9e20718662b6e183a64ee7bf7b841f029881 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 10 Sep 2022 12:58:14 +0900
Subject: [PATCH 2358/3199] save

---
 runtime/src/bank.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index da2e9ed0a10f21..a2b8552081c97e 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1243,7 +1243,7 @@ impl Default for Scheduler {
         let (processed_ee_sender, processed_ee_receiver) = crossbeam_channel::unbounded();
         let (retired_ee_sender, retired_ee_receiver) = crossbeam_channel::unbounded();
 
-        let bank = Arc::new(std::sync::RwLock::new(None::<std::sync::Weak<Bank>>));
+        let bank = Arc::new(None::<std::sync::Weak<Bank>>);
 
         let executing_thread_count = std::env::var("EXECUTING_THREAD_COUNT")
             .unwrap_or(format!("{}", 1))

From f7bfcc05bcb2a141072ecafcbdd49949550d7cee Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 10 Sep 2022 12:58:38 +0900
Subject: [PATCH 2359/3199] save

---
 runtime/src/bank.rs | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index a2b8552081c97e..d91be0b631e5ca 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1271,9 +1271,10 @@ impl Default for Scheduler {
                 execute_time += 1;
                 trace!("execute_substage: thread: {} transaction_index: {} execute_clock: {}", thx, ee.task.transaction_index_in_entries_for_replay(), current_execute_clock);
 
-                let ro_bank = bank.read().unwrap();
-                let weak_bank = ro_bank.as_ref().unwrap().upgrade();
-                let bank = weak_bank.as_ref().unwrap();
+                panic!();
+                //let ro_bank = bank.read().unwrap();
+                //let weak_bank = ro_bank.as_ref().unwrap().upgrade();
+                //let bank = weak_bank.as_ref().unwrap();
 
                 let tx_account_lock_limit = bank.get_transaction_account_lock_limit();
                 let lock_result = ee.task.tx.0

From 426eda7fb1f508d2e0586cd263c2b6afb6383548 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 10 Sep 2022 12:59:16 +0900
Subject: [PATCH 2360/3199] save

---
 runtime/src/bank.rs | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index d91be0b631e5ca..a1021098baa8bb 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1272,9 +1272,9 @@ impl Default for Scheduler {
                 trace!("execute_substage: thread: {} transaction_index: {} execute_clock: {}", thx, ee.task.transaction_index_in_entries_for_replay(), current_execute_clock);
 
                 panic!();
-                //let ro_bank = bank.read().unwrap();
-                //let weak_bank = ro_bank.as_ref().unwrap().upgrade();
-                //let bank = weak_bank.as_ref().unwrap();
+                let ro_bank = bank.unwrap();
+                let weak_bank = ro_bank.as_ref().unwrap().upgrade();
+                let bank = weak_bank.as_ref().unwrap();
 
                 let tx_account_lock_limit = bank.get_transaction_account_lock_limit();
                 let lock_result = ee.task.tx.0

From c40bc26663232cf782adab94cbd3225e5a75644c Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 10 Sep 2022 12:59:32 +0900
Subject: [PATCH 2361/3199] save

---
 runtime/src/bank.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index a1021098baa8bb..26c397f4ba6df3 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1273,7 +1273,7 @@ impl Default for Scheduler {
 
                 panic!();
                 let ro_bank = bank.unwrap();
-                let weak_bank = ro_bank.as_ref().unwrap().upgrade();
+                let weak_bank = ro_bank.unwrap().upgrade();
                 let bank = weak_bank.as_ref().unwrap();
 
                 let tx_account_lock_limit = bank.get_transaction_account_lock_limit();

From ce093eb2cdd6a32c16c92b6d737a7a3f09cf9dc9 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 10 Sep 2022 12:59:42 +0900
Subject: [PATCH 2362/3199] save

---
 runtime/src/bank.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 26c397f4ba6df3..6d06b06181986e 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1273,7 +1273,7 @@ impl Default for Scheduler {
 
                 panic!();
                 let ro_bank = bank.unwrap();
-                let weak_bank = ro_bank.unwrap().upgrade();
+                let weak_bank = ro_bank.upgrade();
                 let bank = weak_bank.as_ref().unwrap();
 
                 let tx_account_lock_limit = bank.get_transaction_account_lock_limit();

From b1d62df9fda5427d752b9deb878581194b5a3e78 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 10 Sep 2022 13:00:06 +0900
Subject: [PATCH 2363/3199] save

---
 runtime/src/bank.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 6d06b06181986e..d5131d025a33ea 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1205,7 +1205,7 @@ struct Scheduler {
     preloader: Arc<solana_scheduler::Preloader>,
     graceful_stop_initiated: bool,
     errors: Arc<std::sync::Mutex<Vec<Result<()>>>>,
-    bank: std::sync::Arc<std::sync::RwLock<std::option::Option<std::sync::Weak<Bank>>>>,
+    bank: std::sync::Arc<std::option::Option<std::sync::Weak<Bank>>>,
     transaction_index: AtomicUsize,
 }
 

From 339158f9e3e36cb7f21d720d267e5d0601dac213 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 10 Sep 2022 13:00:21 +0900
Subject: [PATCH 2364/3199] save

---
 runtime/src/bank.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index d5131d025a33ea..3dd93e5b5b0ac5 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -6447,7 +6447,7 @@ impl Bank {
         let scheduler = {
             let r = self.scheduler.read().unwrap();
 
-            if r.bank.read().unwrap().is_none() {
+            if r.bank.is_none() {
                 drop(r);
                 let w = self.scheduler.write().unwrap();
                 *w.bank.write().unwrap() = Some(Arc::downgrade(&bank));

From 67922e3022481e0be2ad2670fb838d9eec9cf8cd Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 10 Sep 2022 13:00:39 +0900
Subject: [PATCH 2365/3199] save

---
 runtime/src/bank.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 3dd93e5b5b0ac5..181e0db63cfacf 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -6450,7 +6450,7 @@ impl Bank {
             if r.bank.is_none() {
                 drop(r);
                 let w = self.scheduler.write().unwrap();
-                *w.bank.write().unwrap() = Some(Arc::downgrade(&bank));
+                *w.bank = Some(Arc::downgrade(&bank));
                 drop(w);
                 self.scheduler.read().unwrap()
             } else {

From 2d66d5aab06f822a0629d4ebbd556d5d5de94ff4 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 10 Sep 2022 13:01:18 +0900
Subject: [PATCH 2366/3199] save

---
 runtime/src/bank.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 181e0db63cfacf..c854ffbef33b60 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -6449,7 +6449,7 @@ impl Bank {
 
             if r.bank.is_none() {
                 drop(r);
-                let w = self.scheduler.write().unwrap();
+                let mut w = self.scheduler.write().unwrap();
                 *w.bank = Some(Arc::downgrade(&bank));
                 drop(w);
                 self.scheduler.read().unwrap()

From a0e0e02e19560de808456e4f5f35bc41c2a03c20 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 10 Sep 2022 13:05:30 +0900
Subject: [PATCH 2367/3199] save

---
 runtime/src/bank.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index c854ffbef33b60..18b029fd9a26d6 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1205,7 +1205,7 @@ struct Scheduler {
     preloader: Arc<solana_scheduler::Preloader>,
     graceful_stop_initiated: bool,
     errors: Arc<std::sync::Mutex<Vec<Result<()>>>>,
-    bank: std::sync::Arc<std::option::Option<std::sync::Weak<Bank>>>,
+    bank: std::sync::ArcSwapOption<std::sync::Weak<Bank>>,
     transaction_index: AtomicUsize,
 }
 

From 87f5d6ce8cfb1a335cb1734fa3b6b5db44b66e39 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 10 Sep 2022 13:05:45 +0900
Subject: [PATCH 2368/3199] save

---
 runtime/src/bank.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 18b029fd9a26d6..78af324000b992 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1205,7 +1205,7 @@ struct Scheduler {
     preloader: Arc<solana_scheduler::Preloader>,
     graceful_stop_initiated: bool,
     errors: Arc<std::sync::Mutex<Vec<Result<()>>>>,
-    bank: std::sync::ArcSwapOption<std::sync::Weak<Bank>>,
+    bank: arcswap::ArcSwapOption<std::sync::Weak<Bank>>,
     transaction_index: AtomicUsize,
 }
 

From b95b3fea150b5447a885208fd3e6a6b70e02772d Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 10 Sep 2022 13:06:37 +0900
Subject: [PATCH 2369/3199] save

---
 runtime/Cargo.toml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/runtime/Cargo.toml b/runtime/Cargo.toml
index 8b2ea6ce652572..38fb0e02f1e95f 100644
--- a/runtime/Cargo.toml
+++ b/runtime/Cargo.toml
@@ -10,6 +10,7 @@ documentation = "https://docs.rs/solana-runtime"
 edition = "2021"
 
 [dependencies]
+arcswap = "1.5.0"
 arrayref = "0.3.6"
 bincode = "1.3.3"
 blake3 = "1.3.1"

From 9e7350c86968b3aba2ad1753ec62e6a9b631d897 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 10 Sep 2022 13:07:16 +0900
Subject: [PATCH 2370/3199] save

---
 runtime/Cargo.toml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/Cargo.toml b/runtime/Cargo.toml
index 38fb0e02f1e95f..2430ea5e97091b 100644
--- a/runtime/Cargo.toml
+++ b/runtime/Cargo.toml
@@ -10,7 +10,7 @@ documentation = "https://docs.rs/solana-runtime"
 edition = "2021"
 
 [dependencies]
-arcswap = "1.5.0"
+arc-swap = "1.5.0"
 arrayref = "0.3.6"
 bincode = "1.3.3"
 blake3 = "1.3.1"

From 218c99f882f2e418b58cace59f3ef26218fb5e63 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 10 Sep 2022 13:07:30 +0900
Subject: [PATCH 2371/3199] save

---
 Cargo.lock          | 1 +
 runtime/src/bank.rs | 2 +-
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/Cargo.lock b/Cargo.lock
index 5c03faca5f4bc1..ea78b04e08816f 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -6127,6 +6127,7 @@ dependencies = [
 name = "solana-runtime"
 version = "1.12.0"
 dependencies = [
+ "arc-swap",
  "arrayref",
  "assert_matches",
  "bincode",
diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 78af324000b992..a81c50071d030c 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1205,7 +1205,7 @@ struct Scheduler {
     preloader: Arc<solana_scheduler::Preloader>,
     graceful_stop_initiated: bool,
     errors: Arc<std::sync::Mutex<Vec<Result<()>>>>,
-    bank: arcswap::ArcSwapOption<std::sync::Weak<Bank>>,
+    bank: arc_swap::ArcSwapOption<std::sync::Weak<Bank>>,
     transaction_index: AtomicUsize,
 }
 

From 725d93aff6cdffa94882a9b628b80d1440bcb10c Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 10 Sep 2022 13:08:10 +0900
Subject: [PATCH 2372/3199] save

---
 runtime/src/bank.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index a81c50071d030c..511f3e4297c043 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1243,7 +1243,7 @@ impl Default for Scheduler {
         let (processed_ee_sender, processed_ee_receiver) = crossbeam_channel::unbounded();
         let (retired_ee_sender, retired_ee_receiver) = crossbeam_channel::unbounded();
 
-        let bank = Arc::new(None::<std::sync::Weak<Bank>>);
+        let bank = arc_swap::ArcSwapOption::new(None::<std::sync::Weak<Bank>>);
 
         let executing_thread_count = std::env::var("EXECUTING_THREAD_COUNT")
             .unwrap_or(format!("{}", 1))

From 86aa0376e16796e5416ff9d870b03b311a9a9f13 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 10 Sep 2022 13:10:14 +0900
Subject: [PATCH 2373/3199] save

---
 runtime/src/bank.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 511f3e4297c043..7ec89d23920725 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1243,7 +1243,7 @@ impl Default for Scheduler {
         let (processed_ee_sender, processed_ee_receiver) = crossbeam_channel::unbounded();
         let (retired_ee_sender, retired_ee_receiver) = crossbeam_channel::unbounded();
 
-        let bank = arc_swap::ArcSwapOption::new(None::<std::sync::Weak<Bank>>);
+        let bank = arc_swap::ArcSwapAny::new(None::<std::sync::Weak<Bank>>);
 
         let executing_thread_count = std::env::var("EXECUTING_THREAD_COUNT")
             .unwrap_or(format!("{}", 1))

From c163750d28ac813589103c0d448083aeed78fe5a Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 10 Sep 2022 13:15:29 +0900
Subject: [PATCH 2374/3199] save

---
 runtime/Cargo.toml  |  1 -
 runtime/src/bank.rs | 18 ++++++++++--------
 2 files changed, 10 insertions(+), 9 deletions(-)

diff --git a/runtime/Cargo.toml b/runtime/Cargo.toml
index 2430ea5e97091b..8b2ea6ce652572 100644
--- a/runtime/Cargo.toml
+++ b/runtime/Cargo.toml
@@ -10,7 +10,6 @@ documentation = "https://docs.rs/solana-runtime"
 edition = "2021"
 
 [dependencies]
-arc-swap = "1.5.0"
 arrayref = "0.3.6"
 bincode = "1.3.3"
 blake3 = "1.3.1"
diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 7ec89d23920725..7f30d1158ca236 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1205,7 +1205,7 @@ struct Scheduler {
     preloader: Arc<solana_scheduler::Preloader>,
     graceful_stop_initiated: bool,
     errors: Arc<std::sync::Mutex<Vec<Result<()>>>>,
-    bank: arc_swap::ArcSwapOption<std::sync::Weak<Bank>>,
+    bank: std::sync::Arc<std::sync::RwLock<std::option::Option<std::sync::Weak<Bank>>>>,
     transaction_index: AtomicUsize,
 }
 
@@ -1243,7 +1243,7 @@ impl Default for Scheduler {
         let (processed_ee_sender, processed_ee_receiver) = crossbeam_channel::unbounded();
         let (retired_ee_sender, retired_ee_receiver) = crossbeam_channel::unbounded();
 
-        let bank = arc_swap::ArcSwapAny::new(None::<std::sync::Weak<Bank>>);
+        let bank = Arc::new(std::sync::RwLock::new(None::<std::sync::Weak<Bank>>));
 
         let executing_thread_count = std::env::var("EXECUTING_THREAD_COUNT")
             .unwrap_or(format!("{}", 1))
@@ -1271,9 +1271,8 @@ impl Default for Scheduler {
                 execute_time += 1;
                 trace!("execute_substage: thread: {} transaction_index: {} execute_clock: {}", thx, ee.task.transaction_index_in_entries_for_replay(), current_execute_clock);
 
-                panic!();
-                let ro_bank = bank.unwrap();
-                let weak_bank = ro_bank.upgrade();
+                let ro_bank = bank.read().unwrap();
+                let weak_bank = ro_bank.as_ref().unwrap().upgrade();
                 let bank = weak_bank.as_ref().unwrap();
 
                 let tx_account_lock_limit = bank.get_transaction_account_lock_limit();
@@ -6447,10 +6446,13 @@ impl Bank {
         let scheduler = {
             let r = self.scheduler.read().unwrap();
 
-            if r.bank.is_none() {
+            if r.bank.read().unwrap().is_none() {
                 drop(r);
-                let mut w = self.scheduler.write().unwrap();
-                *w.bank = Some(Arc::downgrade(&bank));
+                // this is racy here; we want parking_lot's upgrade; but overwriting should be
+                // safe.
+                let w = self.scheduler.write().unwrap();
+                *w.bank.write().unwrap() = Some(Arc::downgrade(&bank));
+                w.slot = Some(bank.slot);
                 drop(w);
                 self.scheduler.read().unwrap()
             } else {

From f798b0f4a8e249396d404aabbd38eed96baa2b53 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 10 Sep 2022 13:16:40 +0900
Subject: [PATCH 2375/3199] save

---
 Cargo.lock          | 1 -
 runtime/src/bank.rs | 2 ++
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/Cargo.lock b/Cargo.lock
index ea78b04e08816f..5c03faca5f4bc1 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -6127,7 +6127,6 @@ dependencies = [
 name = "solana-runtime"
 version = "1.12.0"
 dependencies = [
- "arc-swap",
  "arrayref",
  "assert_matches",
  "bincode",
diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 7f30d1158ca236..d03163f30e566f 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -6456,6 +6456,8 @@ impl Bank {
                 drop(w);
                 self.scheduler.read().unwrap()
             } else {
+                assert_eq!(bank.slot(), batch.bank.slot());
+                assert_eq!(bank.slot(), r.slot);
                 r
             }
         };

From 5f472040f02a858b3166c90df30a57c86d041573 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 10 Sep 2022 13:16:52 +0900
Subject: [PATCH 2376/3199] save

---
 runtime/src/bank.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index d03163f30e566f..455bc4b3901928 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -6456,7 +6456,7 @@ impl Bank {
                 drop(w);
                 self.scheduler.read().unwrap()
             } else {
-                assert_eq!(bank.slot(), batch.bank.slot());
+                assert_eq!(bank.slot(), batch.bank().slot());
                 assert_eq!(bank.slot(), r.slot);
                 r
             }

From dbd90d50e4b9f193bf85e4e3d0eb10d07b928957 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 10 Sep 2022 13:17:14 +0900
Subject: [PATCH 2377/3199] save

---
 runtime/src/bank.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 455bc4b3901928..1d342518f782d4 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -6457,7 +6457,7 @@ impl Bank {
                 self.scheduler.read().unwrap()
             } else {
                 assert_eq!(bank.slot(), batch.bank().slot());
-                assert_eq!(bank.slot(), r.slot);
+                assert_eq!(bank.slot(), r.slot.unwrap());
                 r
             }
         };

From dc9bcdf9691983bf000af55c103e9180bb588da2 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 10 Sep 2022 13:17:35 +0900
Subject: [PATCH 2378/3199] save

---
 runtime/src/bank.rs | 1 +
 1 file changed, 1 insertion(+)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 1d342518f782d4..f677d530ec4187 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1207,6 +1207,7 @@ struct Scheduler {
     errors: Arc<std::sync::Mutex<Vec<Result<()>>>>,
     bank: std::sync::Arc<std::sync::RwLock<std::option::Option<std::sync::Weak<Bank>>>>,
     transaction_index: AtomicUsize,
+    slot: Option<Slot>,
 }
 
 impl Scheduler {

From 2c614fbec62e5e8c35809d0cca394aebbf864f82 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 10 Sep 2022 13:18:08 +0900
Subject: [PATCH 2379/3199] save

---
 runtime/src/bank.rs | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index f677d530ec4187..b052575f2b933d 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1206,7 +1206,6 @@ struct Scheduler {
     graceful_stop_initiated: bool,
     errors: Arc<std::sync::Mutex<Vec<Result<()>>>>,
     bank: std::sync::Arc<std::sync::RwLock<std::option::Option<std::sync::Weak<Bank>>>>,
-    transaction_index: AtomicUsize,
     slot: Option<Slot>,
 }
 
@@ -1380,7 +1379,7 @@ impl Default for Scheduler {
             graceful_stop_initiated: Default::default(),
             errors,
             bank,
-            transaction_index: Default::default(),
+            slot: Default::default(),
         }
     }
 }

From eec7ebfcf96a6ca306521a6da619b21156ce13a3 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 10 Sep 2022 13:18:28 +0900
Subject: [PATCH 2380/3199] save

---
 runtime/src/bank.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index b052575f2b933d..186f5c8a67da03 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -6452,7 +6452,7 @@ impl Bank {
                 // safe.
                 let w = self.scheduler.write().unwrap();
                 *w.bank.write().unwrap() = Some(Arc::downgrade(&bank));
-                w.slot = Some(bank.slot);
+                *w.slot = Some(bank.slot);
                 drop(w);
                 self.scheduler.read().unwrap()
             } else {

From f5a96ee652697364da45352da7fc14b72f47c6e3 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 10 Sep 2022 13:18:42 +0900
Subject: [PATCH 2381/3199] save

---
 runtime/src/bank.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 186f5c8a67da03..879b2819049cee 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -6450,9 +6450,9 @@ impl Bank {
                 drop(r);
                 // this is racy here; we want parking_lot's upgrade; but overwriting should be
                 // safe.
-                let w = self.scheduler.write().unwrap();
+                let mut w = self.scheduler.write().unwrap();
                 *w.bank.write().unwrap() = Some(Arc::downgrade(&bank));
-                *w.slot = Some(bank.slot);
+                w.slot = Some(bank.slot);
                 drop(w);
                 self.scheduler.read().unwrap()
             } else {

From 87a5aa92369b012e3c7db6ba2904515f1ac0cb17 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 10 Sep 2022 13:21:33 +0900
Subject: [PATCH 2382/3199] save

---
 runtime/src/bank.rs | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 879b2819049cee..f13bfe230a800e 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -6443,6 +6443,8 @@ impl Bank {
         bank: &Arc<Bank>,
         transaction_indexes: &Vec<usize>,
     ) {
+        assert_eq!(bank.slot(), batch.bank().slot());
+
         let scheduler = {
             let r = self.scheduler.read().unwrap();
 
@@ -6456,7 +6458,6 @@ impl Bank {
                 drop(w);
                 self.scheduler.read().unwrap()
             } else {
-                assert_eq!(bank.slot(), batch.bank().slot());
                 assert_eq!(bank.slot(), r.slot.unwrap());
                 r
             }

From 15e4953ba82eb786b8c86a9a1deecf1d67f98749 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 10 Sep 2022 13:22:22 +0900
Subject: [PATCH 2383/3199] save

---
 runtime/src/bank.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index f13bfe230a800e..ded1369c5b4cd9 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -6450,7 +6450,7 @@ impl Bank {
 
             if r.bank.read().unwrap().is_none() {
                 drop(r);
-                // this is racy here; we want parking_lot's upgrade; but overwriting should be
+                // this relocking (r=>w) is racy here; we want parking_lot's upgrade; but overwriting should be
                 // safe.
                 let mut w = self.scheduler.write().unwrap();
                 *w.bank.write().unwrap() = Some(Arc::downgrade(&bank));

From b093b4aeb5ef29f29b98c813f1bfe962ce65cb6d Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 10 Sep 2022 13:48:42 +0900
Subject: [PATCH 2384/3199] save

---
 scheduler/src/lib.rs | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index aaecf908b2d8e8..485898112faa49 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1541,7 +1541,7 @@ impl ScheduleStage {
                 while executing_queue_count + provisioning_tracker_count < max_executing_queue_count {
                     let prefer_immediate = true; //provisioning_tracker_count / 4 > executing_queue_count;
 
-                    if let Some(ee) = Self::schedule_next_execution(
+                    let maybe_ee = Self::schedule_next_execution(
                         ast,
                         &task_sender,
                         runnable_queue,
@@ -1553,11 +1553,15 @@ impl ScheduleStage {
                         &mut execute_clock,
                         &mut provisioning_tracker_count,
                         true
-                    ) {
+                    )
+                    if let Some(ee) = maybe_ee {
                         executing_queue_count = executing_queue_count.checked_add(1).unwrap();
                         to_execute_substage.send(ExecutablePayload(ee)).unwrap();
+                    } else {
+                        break;
                     }
                     debug!("schedule_once id_{:016x} [R] ch(prev: {}, exec: {}|{}), r: {}, u/c: {}/{}, (imm+provi)/max: ({}+{})/{} s: {} done: {}", random_id, from_prev.len(), to_execute_substage.len(), from_exec.len(), runnable_queue.task_count(), address_book.uncontended_task_ids.len(), contended_count, executing_queue_count, provisioning_tracker_count, max_executing_queue_count, address_book.stuck_tasks.len(), processed_count);
+
                     interval_count += 1;
                     if interval_count % 100 == 0 {
                         let elapsed = start.elapsed();

From 563ea19ace8608a95cba37ce958e860ac8e13f69 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 10 Sep 2022 13:48:52 +0900
Subject: [PATCH 2385/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 485898112faa49..a77eb41d737998 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1553,7 +1553,7 @@ impl ScheduleStage {
                         &mut execute_clock,
                         &mut provisioning_tracker_count,
                         true
-                    )
+                    );
                     if let Some(ee) = maybe_ee {
                         executing_queue_count = executing_queue_count.checked_add(1).unwrap();
                         to_execute_substage.send(ExecutablePayload(ee)).unwrap();

From 692be1bd8882b4b8ba8d4793da3bf71b91a55fb6 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 10 Sep 2022 13:53:14 +0900
Subject: [PATCH 2386/3199] save

---
 scheduler/src/lib.rs | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index a77eb41d737998..e0f5f059990e7b 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1557,10 +1557,11 @@ impl ScheduleStage {
                     if let Some(ee) = maybe_ee {
                         executing_queue_count = executing_queue_count.checked_add(1).unwrap();
                         to_execute_substage.send(ExecutablePayload(ee)).unwrap();
+                        debug!("schedule_once id_{:016x} [R] ch(prev: {}, exec: {}|{}), r: {}, u/c: {}/{}, (imm+provi)/max: ({}+{})/{} s: {} done: {}", random_id, from_prev.len(), to_execute_substage.len(), from_exec.len(), runnable_queue.task_count(), address_book.uncontended_task_ids.len(), contended_count, executing_queue_count, provisioning_tracker_count, max_executing_queue_count, address_book.stuck_tasks.len(), processed_count);
                     } else {
+                        debug!("schedule_once id_{:016x} [R] ch(prev: {}, exec: {}|{}), r: {}, u/c: {}/{}, (imm+provi)/max: ({}+{})/{} s: {} done: {}", random_id, from_prev.len(), to_execute_substage.len(), from_exec.len(), runnable_queue.task_count(), address_book.uncontended_task_ids.len(), contended_count, executing_queue_count, provisioning_tracker_count, max_executing_queue_count, address_book.stuck_tasks.len(), processed_count);
                         break;
                     }
-                    debug!("schedule_once id_{:016x} [R] ch(prev: {}, exec: {}|{}), r: {}, u/c: {}/{}, (imm+provi)/max: ({}+{})/{} s: {} done: {}", random_id, from_prev.len(), to_execute_substage.len(), from_exec.len(), runnable_queue.task_count(), address_book.uncontended_task_ids.len(), contended_count, executing_queue_count, provisioning_tracker_count, max_executing_queue_count, address_book.stuck_tasks.len(), processed_count);
 
                     interval_count += 1;
                     if interval_count % 100 == 0 {

From 16381114eb5f02bea6f93742b23c087ccc8e5148 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 10 Sep 2022 14:08:46 +0900
Subject: [PATCH 2387/3199] save

---
 scheduler/src/lib.rs | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index e0f5f059990e7b..b5c9e0c419edfe 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1533,7 +1533,7 @@ impl ScheduleStage {
                         if elapsed > std::time::Duration::from_millis(150) {
                             let delta = (processed_count - last_processed_count) as u128;
                             let elapsed2 = elapsed.as_micros();
-                            info!("schedule_once:interval id_{:016x} ch(prev: {}, exec: {}|{}), r: {}, u/c: {}/{}, (imm+provi)/max: ({}+{})/{} s: {} done: {} ({}txs/{}us={}tps)", random_id, from_prev.len(), to_execute_substage.len(), from_exec.len(), runnable_queue.task_count(), address_book.uncontended_task_ids.len(), contended_count, executing_queue_count, provisioning_tracker_count, max_executing_queue_count, address_book.stuck_tasks.len(), processed_count, delta, elapsed.as_micros(), 1000_u128*delta/elapsed2);
+                            info!("schedule_once:interval id_{:016x} ch(prev: {}, exec: {}|{}), r: {}, u/c: {}/{}, (imm+provi)/max: ({}+{})/{} s: {} done: {} ({}txs/{}us={}tps)", random_id, from_prev.len(), to_execute_substage.len(), from_exec.len(), runnable_queue.task_count(), address_book.uncontended_task_ids.len(), contended_count, executing_queue_count, provisioning_tracker_count, max_executing_queue_count, address_book.stuck_tasks.len(), processed_count, delta, elapsed.as_micros(), 1_000_000_u128*delta/elapsed2);
                             (start, last_processed_count) = (std::time::Instant::now(), processed_count);
                         }
                     }
@@ -1562,6 +1562,10 @@ impl ScheduleStage {
                         debug!("schedule_once id_{:016x} [R] ch(prev: {}, exec: {}|{}), r: {}, u/c: {}/{}, (imm+provi)/max: ({}+{})/{} s: {} done: {}", random_id, from_prev.len(), to_execute_substage.len(), from_exec.len(), runnable_queue.task_count(), address_book.uncontended_task_ids.len(), contended_count, executing_queue_count, provisioning_tracker_count, max_executing_queue_count, address_book.stuck_tasks.len(), processed_count);
                         break;
                     }
+                    if !from_exec.is_empty() {
+                        trace!("abort aggressive readable queue processing due to non-empty from_exec");
+                        break;
+                    }
 
                     interval_count += 1;
                     if interval_count % 100 == 0 {
@@ -1569,14 +1573,10 @@ impl ScheduleStage {
                         if elapsed > std::time::Duration::from_millis(150) {
                             let delta = (processed_count - last_processed_count) as u128;
                             let elapsed2 = elapsed.as_micros();
-                            info!("schedule_once:interval id_{:016x} ch(prev: {}, exec: {}|{}), r: {}, u/c: {}/{}, (imm+provi)/max: ({}+{})/{} s: {} done: {} ({}txs/{}us={}tps)", random_id, from_prev.len(), to_execute_substage.len(), from_exec.len(), runnable_queue.task_count(), address_book.uncontended_task_ids.len(), contended_count, executing_queue_count, provisioning_tracker_count, max_executing_queue_count, address_book.stuck_tasks.len(), processed_count, delta, elapsed.as_micros(), 1000_u128*delta/elapsed2);
+                            info!("schedule_once:interval id_{:016x} ch(prev: {}, exec: {}|{}), r: {}, u/c: {}/{}, (imm+provi)/max: ({}+{})/{} s: {} done: {} ({}txs/{}us={}tps)", random_id, from_prev.len(), to_execute_substage.len(), from_exec.len(), runnable_queue.task_count(), address_book.uncontended_task_ids.len(), contended_count, executing_queue_count, provisioning_tracker_count, max_executing_queue_count, address_book.stuck_tasks.len(), processed_count, delta, elapsed.as_micros(), 1_000_000_u128*delta/elapsed2);
                             (start, last_processed_count) = (std::time::Instant::now(), processed_count);
                         }
                     }
-                    if !from_exec.is_empty() {
-                        trace!("abort aggressive readable queue processing due to non-empty from_exec");
-                        break;
-                    }
                 }
 
                 if first_iteration {

From ccf881e73d1524d3c7479a1be80c83a69b9d0f48 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 10 Sep 2022 14:45:41 +0900
Subject: [PATCH 2388/3199] save

---
 metrics/src/metrics.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/metrics/src/metrics.rs b/metrics/src/metrics.rs
index 4b9b183de98ee4..99d6335e8cb7d1 100644
--- a/metrics/src/metrics.rs
+++ b/metrics/src/metrics.rs
@@ -160,7 +160,7 @@ impl Default for MetricsAgent {
 
         Self::new(
             Arc::new(InfluxDbMetricsWriter::new()),
-            Duration::from_secs(10),
+            Duration::from_secs(1),
             max_points_per_sec,
         )
     }

From b46197979a1f7d84dff141fea29469f00f3c6d38 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 10 Sep 2022 17:12:13 +0900
Subject: [PATCH 2389/3199] save

---
 runtime/src/bank.rs | 8 +++-----
 1 file changed, 3 insertions(+), 5 deletions(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index ded1369c5b4cd9..aaac30357b0197 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1251,7 +1251,6 @@ impl Default for Scheduler {
             .unwrap();
 
         let send_metrics = std::env::var("SEND_METRICS").is_ok();
-        let disable_clock_asserts = std::env::var("DISABLE_CLOCK_ASSERTS").is_ok();
 
         let executing_thread_handles = (0..executing_thread_count).map(|thx| {
             let (scheduled_ee_receiver, processed_ee_sender) = (scheduled_ee_receiver.clone(), processed_ee_sender.clone());
@@ -1265,9 +1264,6 @@ impl Default for Scheduler {
                 let mut process_message_time = Measure::start("process_message_time");
 
                 let current_execute_clock = ee.task.execute_time();
-                if !disable_clock_asserts {
-                    assert_eq!(current_execute_clock, execute_time);
-                }
                 execute_time += 1;
                 trace!("execute_substage: thread: {} transaction_index: {} execute_clock: {}", thx, ee.task.transaction_index_in_entries_for_replay(), current_execute_clock);
 
@@ -1294,6 +1290,8 @@ impl Default for Scheduler {
                     None
                 );
                 drop(bank);
+                drop(weak_bank);
+                drop(ro_bank);
 
                 let TransactionResults {
                     fee_collection_results,
@@ -1315,7 +1313,7 @@ impl Default for Scheduler {
 
                     datapoint_info!(
                         "individual_tx_stats",
-                        ("slot", bank.slot(), i64),
+                        ("slot", bank.slot(), i64), // what? why dropped bank can be used here??
                         ("thread", current_thread_name, String),
                         ("signature", &sig, String),
                         ("account_locks_in_json", "{}", String),

From b259d37acc9720f152682c3ce7d59ac26d0912e4 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 10 Sep 2022 17:12:37 +0900
Subject: [PATCH 2390/3199] save

---
 runtime/src/bank.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index aaac30357b0197..d387ec7069100b 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1290,8 +1290,8 @@ impl Default for Scheduler {
                     None
                 );
                 drop(bank);
-                drop(weak_bank);
-                drop(ro_bank);
+                //drop(weak_bank);
+                //drop(ro_bank);
 
                 let TransactionResults {
                     fee_collection_results,

From 76e30ee0311f2a8896bb676ad699cd6b82c4a091 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 10 Sep 2022 17:13:11 +0900
Subject: [PATCH 2391/3199] save

---
 runtime/src/bank.rs | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index d387ec7069100b..d872dee287cb04 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1270,6 +1270,7 @@ impl Default for Scheduler {
                 let ro_bank = bank.read().unwrap();
                 let weak_bank = ro_bank.as_ref().unwrap().upgrade();
                 let bank = weak_bank.as_ref().unwrap();
+                let slot = bank.slot();
 
                 let tx_account_lock_limit = bank.get_transaction_account_lock_limit();
                 let lock_result = ee.task.tx.0
@@ -1290,8 +1291,8 @@ impl Default for Scheduler {
                     None
                 );
                 drop(bank);
-                //drop(weak_bank);
-                //drop(ro_bank);
+                drop(weak_bank);
+                drop(ro_bank);
 
                 let TransactionResults {
                     fee_collection_results,
@@ -1313,7 +1314,7 @@ impl Default for Scheduler {
 
                     datapoint_info!(
                         "individual_tx_stats",
-                        ("slot", bank.slot(), i64), // what? why dropped bank can be used here??
+                        ("slot", slot, i64), // what? why dropped bank can be used here??
                         ("thread", current_thread_name, String),
                         ("signature", &sig, String),
                         ("account_locks_in_json", "{}", String),

From 16f176b6ca67f90234cbcb4eb3305a7c1c44757c Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 10 Sep 2022 17:13:53 +0900
Subject: [PATCH 2392/3199] save

---
 runtime/src/bank.rs | 1 +
 1 file changed, 1 insertion(+)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index d872dee287cb04..942d3ebe33dea0 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1293,6 +1293,7 @@ impl Default for Scheduler {
                 drop(bank);
                 drop(weak_bank);
                 drop(ro_bank);
+                drop(batch);
 
                 let TransactionResults {
                     fee_collection_results,

From 7f893e481b08fed54ce5d048c9c7daebdb59f746 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 10 Sep 2022 17:14:00 +0900
Subject: [PATCH 2393/3199] save

---
 runtime/src/bank.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 942d3ebe33dea0..0f6c97ba9c90ae 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1291,9 +1291,9 @@ impl Default for Scheduler {
                     None
                 );
                 drop(bank);
+                drop(batch);
                 drop(weak_bank);
                 drop(ro_bank);
-                drop(batch);
 
                 let TransactionResults {
                     fee_collection_results,

From fe5ffa8bc6b138d004c5075c73058267b47a25e2 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 10 Sep 2022 17:14:58 +0900
Subject: [PATCH 2394/3199] save

---
 runtime/src/bank.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 0f6c97ba9c90ae..51d7d3a38716b8 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1315,7 +1315,7 @@ impl Default for Scheduler {
 
                     datapoint_info!(
                         "individual_tx_stats",
-                        ("slot", slot, i64), // what? why dropped bank can be used here??
+                        ("slot", slot, i64),
                         ("thread", current_thread_name, String),
                         ("signature", &sig, String),
                         ("account_locks_in_json", "{}", String),

From e37f496df997d1b9ac7f39e3ec74be5af38a3727 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 10 Sep 2022 21:19:20 +0900
Subject: [PATCH 2395/3199] save

---
 runtime/src/bank.rs  | 19 +++++++++++++------
 scheduler/src/lib.rs | 21 +++++++++++++++++++++
 2 files changed, 34 insertions(+), 6 deletions(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 51d7d3a38716b8..b17c07687501fe 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1301,12 +1301,15 @@ impl Default for Scheduler {
                     ..
                 } = tx_results;
 
-                ee.execution_result = Some(fee_collection_results.into_iter().collect::<Result<_>>());
-                let details = execution_results[0].details().unwrap();
-                ee.cu = details.executed_units;
-
+                let tx_result = fee_collection_results.into_iter().collect::<Result<_>>();
+                let status_str = if tx_result.is_ok() {
+                    let details = execution_results[0].details().unwrap();
+                    ee.cu = details.executed_units;
+                    send_metrics.and_then(|| format!("{:?}", details.status))
+                } else {
+                    send_metrics.and_then(|| format!("{:?}", ts_result))
+                }
 
-                //ee.reindex_with_address_book();
                 if send_metrics {
                     let sig = ee.task.tx.0.signature().to_string();
 
@@ -1319,11 +1322,15 @@ impl Default for Scheduler {
                         ("thread", current_thread_name, String),
                         ("signature", &sig, String),
                         ("account_locks_in_json", "{}", String),
-                        ("status", format!("{:?}", details.status), String),
+                        ("status", format!("{:?}", status_str.unwrap()), String),
                         ("duration", duration_with_overhead, i64),
                         ("compute_units", ee.cu, i64),
                     );
                 }
+
+                ee.execution_result = Some(tx_result);
+
+                //ee.reindex_with_address_book();
                 processed_ee_sender.send(solana_scheduler::UnlockablePayload(ee)).unwrap();
             }
             Ok(())
diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index b5c9e0c419edfe..dd4ef222177ebe 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -874,6 +874,27 @@ enum TaskSource {
     Stuck,
 }
 
+enum SelectionContext {
+    Runnable,
+    Contended(u8, Vec<TaskInQueue>),
+}
+
+impl SelectionContext {
+    fn should_continue(&self) -> bool {
+        match(self) {
+            SelectionContext::Runnable => true,
+            SelectionContext::Contended(failure_count) => failure_count < 2,
+        }
+    }
+
+    fn runnable_exclusive(&self) -> bool {
+        match(self) {
+            SelectionContext::Runnable => true,
+            SelectionContext::Contended(_) => false,
+        }
+    }
+}
+
 impl ScheduleStage {
     fn push_to_runnable_queue(task: TaskInQueue, runnable_queue: &mut TaskQueue) {
         runnable_queue.add_to_schedule(task.unique_weight, task);

From ff60cbfae3d69942f48ff0a0fe485f3d0fd580bd Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 10 Sep 2022 21:19:49 +0900
Subject: [PATCH 2396/3199] save

---
 scheduler/src/lib.rs | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index dd4ef222177ebe..fd5474051c54fc 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -874,6 +874,7 @@ enum TaskSource {
     Stuck,
 }
 
+/*
 enum SelectionContext {
     Runnable,
     Contended(u8, Vec<TaskInQueue>),
@@ -894,6 +895,7 @@ impl SelectionContext {
         }
     }
 }
+*/
 
 impl ScheduleStage {
     fn push_to_runnable_queue(task: TaskInQueue, runnable_queue: &mut TaskQueue) {

From 8ba226e1766f6d32d9f9ca8ef33dd9bb4bcb4d5c Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 10 Sep 2022 21:19:57 +0900
Subject: [PATCH 2397/3199] save

---
 runtime/src/bank.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index b17c07687501fe..822e39bbf5136c 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1308,7 +1308,7 @@ impl Default for Scheduler {
                     send_metrics.and_then(|| format!("{:?}", details.status))
                 } else {
                     send_metrics.and_then(|| format!("{:?}", ts_result))
-                }
+                };
 
                 if send_metrics {
                     let sig = ee.task.tx.0.signature().to_string();

From 8756453e752b5367ba38323193c8b98b930c90f1 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 10 Sep 2022 21:20:09 +0900
Subject: [PATCH 2398/3199] save

---
 runtime/src/bank.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 822e39bbf5136c..de8a7998ad30af 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1305,9 +1305,9 @@ impl Default for Scheduler {
                 let status_str = if tx_result.is_ok() {
                     let details = execution_results[0].details().unwrap();
                     ee.cu = details.executed_units;
-                    send_metrics.and_then(|| format!("{:?}", details.status))
+                    send_metrics.then(|| format!("{:?}", details.status))
                 } else {
-                    send_metrics.and_then(|| format!("{:?}", ts_result))
+                    send_metrics.then(|| format!("{:?}", ts_result))
                 };
 
                 if send_metrics {

From 8739f8463b937688927c60bedc2c5e4479e68a20 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 10 Sep 2022 21:20:22 +0900
Subject: [PATCH 2399/3199] save

---
 runtime/src/bank.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index de8a7998ad30af..8c0f3df8fe117a 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1307,7 +1307,7 @@ impl Default for Scheduler {
                     ee.cu = details.executed_units;
                     send_metrics.then(|| format!("{:?}", details.status))
                 } else {
-                    send_metrics.then(|| format!("{:?}", ts_result))
+                    send_metrics.then(|| format!("{:?}", tx_result))
                 };
 
                 if send_metrics {

From ca5487af9b7653e95702666cc4ed1bffff63ea3f Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 10 Sep 2022 21:23:43 +0900
Subject: [PATCH 2400/3199] save

---
 runtime/src/bank.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 8c0f3df8fe117a..7c0a71adbd34d6 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1303,7 +1303,7 @@ impl Default for Scheduler {
 
                 let tx_result = fee_collection_results.into_iter().collect::<Result<_>>();
                 let status_str = if tx_result.is_ok() {
-                    let details = execution_results[0].details().unwrap();
+                    let details = execution_results[0].details().expect(format!("tx_result: {:?}", tx_result));
                     ee.cu = details.executed_units;
                     send_metrics.then(|| format!("{:?}", details.status))
                 } else {

From b75e0629dc9ddf75de2fac6cc4ff5a0a1860ebfa Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 10 Sep 2022 21:23:56 +0900
Subject: [PATCH 2401/3199] save

---
 runtime/src/bank.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 7c0a71adbd34d6..147b9014e551fe 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1303,7 +1303,7 @@ impl Default for Scheduler {
 
                 let tx_result = fee_collection_results.into_iter().collect::<Result<_>>();
                 let status_str = if tx_result.is_ok() {
-                    let details = execution_results[0].details().expect(format!("tx_result: {:?}", tx_result));
+                    let details = execution_results[0].details().expect(&format!("tx_result: {:?}", tx_result));
                     ee.cu = details.executed_units;
                     send_metrics.then(|| format!("{:?}", details.status))
                 } else {

From 9a24562719ebd777339bd634efddbbfa5ecca661 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 10 Sep 2022 21:28:36 +0900
Subject: [PATCH 2402/3199] save

---
 scheduler/src/lib.rs | 15 +++++++++------
 1 file changed, 9 insertions(+), 6 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index fd5474051c54fc..6c2c5537fe1c3b 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1485,7 +1485,8 @@ impl ScheduleStage {
                 })
                 .unwrap()
         }).collect::<Vec<_>>();
-        let (mut start, mut last_processed_count) = (std::time::Instant::now(), 0_usize);
+        let (mut last_time, mut last_processed_count) = (std::time::Instant::now(), 0_usize);
+        let start_time = last_time.copy();
 
         let (mut from_disconnected, mut from_exec_disconnected, mut no_more_work): (bool, bool, bool) = Default::default();
         loop {
@@ -1552,12 +1553,12 @@ impl ScheduleStage {
                     debug!("schedule_once id_{:016x} [C] ch(prev: {}, exec: {}|{}), r: {}, u/c: {}/{}, (imm+provi)/max: ({}+{})/{} s: {} done: {}", random_id, from_prev.len(), to_execute_substage.len(), from_exec.len(), runnable_queue.task_count(), address_book.uncontended_task_ids.len(), contended_count, executing_queue_count, provisioning_tracker_count, max_executing_queue_count, address_book.stuck_tasks.len(), processed_count);
                     interval_count += 1;
                     if interval_count % 100 == 0 {
-                        let elapsed = start.elapsed();
+                        let elapsed = last_time.elapsed();
                         if elapsed > std::time::Duration::from_millis(150) {
                             let delta = (processed_count - last_processed_count) as u128;
                             let elapsed2 = elapsed.as_micros();
                             info!("schedule_once:interval id_{:016x} ch(prev: {}, exec: {}|{}), r: {}, u/c: {}/{}, (imm+provi)/max: ({}+{})/{} s: {} done: {} ({}txs/{}us={}tps)", random_id, from_prev.len(), to_execute_substage.len(), from_exec.len(), runnable_queue.task_count(), address_book.uncontended_task_ids.len(), contended_count, executing_queue_count, provisioning_tracker_count, max_executing_queue_count, address_book.stuck_tasks.len(), processed_count, delta, elapsed.as_micros(), 1_000_000_u128*delta/elapsed2);
-                            (start, last_processed_count) = (std::time::Instant::now(), processed_count);
+                            (last_time, last_processed_count) = (std::time::Instant::now(), processed_count);
                         }
                     }
                 }
@@ -1592,12 +1593,12 @@ impl ScheduleStage {
 
                     interval_count += 1;
                     if interval_count % 100 == 0 {
-                        let elapsed = start.elapsed();
+                        let elapsed = last_time.elapsed();
                         if elapsed > std::time::Duration::from_millis(150) {
                             let delta = (processed_count - last_processed_count) as u128;
                             let elapsed2 = elapsed.as_micros();
                             info!("schedule_once:interval id_{:016x} ch(prev: {}, exec: {}|{}), r: {}, u/c: {}/{}, (imm+provi)/max: ({}+{})/{} s: {} done: {} ({}txs/{}us={}tps)", random_id, from_prev.len(), to_execute_substage.len(), from_exec.len(), runnable_queue.task_count(), address_book.uncontended_task_ids.len(), contended_count, executing_queue_count, provisioning_tracker_count, max_executing_queue_count, address_book.stuck_tasks.len(), processed_count, delta, elapsed.as_micros(), 1_000_000_u128*delta/elapsed2);
-                            (start, last_processed_count) = (std::time::Instant::now(), processed_count);
+                            (last_time, last_processed_count) = (std::time::Instant::now(), processed_count);
                         }
                     }
                 }
@@ -1654,7 +1655,9 @@ impl ScheduleStage {
             indexer_handle.join().unwrap().unwrap();
         }
 
-        info!("schedule_once:final id_{:016x} (from_disconnected: {}, from_exec_disconnected: {}, no_more_work: {}) ch(prev: {}, exec: {}|{}), runnnable: {}, contended: {}, (immediate+provisional)/max: ({}+{})/{} uncontended: {} stuck: {} processed: {}!", random_id, from_disconnected, from_exec_disconnected, no_more_work, from_prev.len(), to_execute_substage.len(), from_exec.len(), runnable_queue.task_count(), contended_count, executing_queue_count, provisioning_tracker_count, max_executing_queue_count, address_book.uncontended_task_ids.len(), address_book.stuck_tasks.len(), processed_count);
+
+        let elapsed2 = start_time.elapsed().as_micros();
+        info!("schedule_once:final id_{:016x} (from_disconnected: {}, from_exec_disconnected: {}, no_more_work: {}) ch(prev: {}, exec: {}|{}), runnnable: {}, contended: {}, (immediate+provisional)/max: ({}+{})/{} uncontended: {} stuck: {} overall: {}txs/{}us={}tps!", random_id, from_disconnected, from_exec_disconnected, no_more_work, from_prev.len(), to_execute_substage.len(), from_exec.len(), runnable_queue.task_count(), contended_count, executing_queue_count, provisioning_tracker_count, max_executing_queue_count, address_book.uncontended_task_ids.len(), address_book.stuck_tasks.len(), processed_count, elapsed2.as_micros(), 1_000_000_u128*processed_count/elapsed2);
     }
 
     pub fn run(

From 3c5a08ee6cdb71b7bb137340d41084c21b16b608 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 10 Sep 2022 21:28:46 +0900
Subject: [PATCH 2403/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 6c2c5537fe1c3b..e5d880d562c426 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1486,7 +1486,7 @@ impl ScheduleStage {
                 .unwrap()
         }).collect::<Vec<_>>();
         let (mut last_time, mut last_processed_count) = (std::time::Instant::now(), 0_usize);
-        let start_time = last_time.copy();
+        let start_time = last_time.clone();
 
         let (mut from_disconnected, mut from_exec_disconnected, mut no_more_work): (bool, bool, bool) = Default::default();
         loop {

From afbe6ff0acae9b36f589fc74cc100db5540a27f1 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 10 Sep 2022 21:29:14 +0900
Subject: [PATCH 2404/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index e5d880d562c426..065c5f1bd7effc 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1657,7 +1657,7 @@ impl ScheduleStage {
 
 
         let elapsed2 = start_time.elapsed().as_micros();
-        info!("schedule_once:final id_{:016x} (from_disconnected: {}, from_exec_disconnected: {}, no_more_work: {}) ch(prev: {}, exec: {}|{}), runnnable: {}, contended: {}, (immediate+provisional)/max: ({}+{})/{} uncontended: {} stuck: {} overall: {}txs/{}us={}tps!", random_id, from_disconnected, from_exec_disconnected, no_more_work, from_prev.len(), to_execute_substage.len(), from_exec.len(), runnable_queue.task_count(), contended_count, executing_queue_count, provisioning_tracker_count, max_executing_queue_count, address_book.uncontended_task_ids.len(), address_book.stuck_tasks.len(), processed_count, elapsed2.as_micros(), 1_000_000_u128*processed_count/elapsed2);
+        info!("schedule_once:final id_{:016x} (from_disconnected: {}, from_exec_disconnected: {}, no_more_work: {}) ch(prev: {}, exec: {}|{}), runnnable: {}, contended: {}, (immediate+provisional)/max: ({}+{})/{} uncontended: {} stuck: {} overall: {}txs/{}us={}tps!", random_id, from_disconnected, from_exec_disconnected, no_more_work, from_prev.len(), to_execute_substage.len(), from_exec.len(), runnable_queue.task_count(), contended_count, executing_queue_count, provisioning_tracker_count, max_executing_queue_count, address_book.uncontended_task_ids.len(), address_book.stuck_tasks.len(), processed_count, elapsed.as_micros(), 1_000_000_u128*processed_count/elapsed2);
     }
 
     pub fn run(

From 6bb20a573b06ab01d3d8d09528404c6060759d9a Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 10 Sep 2022 21:29:45 +0900
Subject: [PATCH 2405/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 065c5f1bd7effc..361702a874c2a4 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1657,7 +1657,7 @@ impl ScheduleStage {
 
 
         let elapsed2 = start_time.elapsed().as_micros();
-        info!("schedule_once:final id_{:016x} (from_disconnected: {}, from_exec_disconnected: {}, no_more_work: {}) ch(prev: {}, exec: {}|{}), runnnable: {}, contended: {}, (immediate+provisional)/max: ({}+{})/{} uncontended: {} stuck: {} overall: {}txs/{}us={}tps!", random_id, from_disconnected, from_exec_disconnected, no_more_work, from_prev.len(), to_execute_substage.len(), from_exec.len(), runnable_queue.task_count(), contended_count, executing_queue_count, provisioning_tracker_count, max_executing_queue_count, address_book.uncontended_task_ids.len(), address_book.stuck_tasks.len(), processed_count, elapsed.as_micros(), 1_000_000_u128*processed_count/elapsed2);
+        info!("schedule_once:final id_{:016x} (from_disconnected: {}, from_exec_disconnected: {}, no_more_work: {}) ch(prev: {}, exec: {}|{}), runnnable: {}, contended: {}, (immediate+provisional)/max: ({}+{})/{} uncontended: {} stuck: {} overall: {}txs/{}us={}tps!", random_id, from_disconnected, from_exec_disconnected, no_more_work, from_prev.len(), to_execute_substage.len(), from_exec.len(), runnable_queue.task_count(), contended_count, executing_queue_count, provisioning_tracker_count, max_executing_queue_count, address_book.uncontended_task_ids.len(), address_book.stuck_tasks.len(), processed_count, elapsed.as_micros(), 1_000_000_u128*(processed_count as u128)/elapsed2);
     }
 
     pub fn run(

From 52faacf74fc15f2bfc483f8c407d4360acb45f5a Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 10 Sep 2022 21:30:20 +0900
Subject: [PATCH 2406/3199] save

---
 scheduler/src/lib.rs | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 361702a874c2a4..b3075382a757d3 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1656,7 +1656,8 @@ impl ScheduleStage {
         }
 
 
-        let elapsed2 = start_time.elapsed().as_micros();
+        let elapsed = start_time.elapsed();
+        let elapsed2 = elapsed.as_micros();
         info!("schedule_once:final id_{:016x} (from_disconnected: {}, from_exec_disconnected: {}, no_more_work: {}) ch(prev: {}, exec: {}|{}), runnnable: {}, contended: {}, (immediate+provisional)/max: ({}+{})/{} uncontended: {} stuck: {} overall: {}txs/{}us={}tps!", random_id, from_disconnected, from_exec_disconnected, no_more_work, from_prev.len(), to_execute_substage.len(), from_exec.len(), runnable_queue.task_count(), contended_count, executing_queue_count, provisioning_tracker_count, max_executing_queue_count, address_book.uncontended_task_ids.len(), address_book.stuck_tasks.len(), processed_count, elapsed.as_micros(), 1_000_000_u128*(processed_count as u128)/elapsed2);
     }
 

From 787b8b351ea5dc6ac65c253f98f54dc23631dd5d Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 10 Sep 2022 21:33:43 +0900
Subject: [PATCH 2407/3199] save

---
 scheduler/src/lib.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index b3075382a757d3..f92940b977556e 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1514,7 +1514,7 @@ impl ScheduleStage {
                        assert_eq!(from_prev.len(), 0);
                        from_disconnected |= true;
                        from_prev = never;
-                       info!("flushing2..: {:?} {} {} {} {}", (from_disconnected, from_exec_disconnected), runnable_queue.task_count(), contended_count, executing_queue_count, provisioning_tracker_count);
+                       trace!("flushing2..: {:?} {} {} {} {}", (from_disconnected, from_exec_disconnected), runnable_queue.task_count(), contended_count, executing_queue_count, provisioning_tracker_count);
                    }
                }
             }
@@ -1550,7 +1550,7 @@ impl ScheduleStage {
                         executing_queue_count = executing_queue_count.checked_add(1).unwrap();
                         to_execute_substage.send(ExecutablePayload(ee)).unwrap();
                     }
-                    debug!("schedule_once id_{:016x} [C] ch(prev: {}, exec: {}|{}), r: {}, u/c: {}/{}, (imm+provi)/max: ({}+{})/{} s: {} done: {}", random_id, from_prev.len(), to_execute_substage.len(), from_exec.len(), runnable_queue.task_count(), address_book.uncontended_task_ids.len(), contended_count, executing_queue_count, provisioning_tracker_count, max_executing_queue_count, address_book.stuck_tasks.len(), processed_count);
+                    debug!("schedule_once id_{:016x} [C] ch(prev: {}, exec: {}|{}), r: {}, u/c: {}/{}, (imm+provi)/max: ({}+{})/{} s: {} done: {}", random_id, (if from_disconnected { "N/A" } else { format!("{}", from_prev.len()) }), to_execute_substage.len(), from_exec.len(), runnable_queue.task_count(), address_book.uncontended_task_ids.len(), contended_count, executing_queue_count, provisioning_tracker_count, max_executing_queue_count, address_book.stuck_tasks.len(), processed_count);
                     interval_count += 1;
                     if interval_count % 100 == 0 {
                         let elapsed = last_time.elapsed();

From 18d089b1d3f550f41ee2471a5f538dccebe605ef Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 10 Sep 2022 21:34:02 +0900
Subject: [PATCH 2408/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index f92940b977556e..1bf2ca2acbdbce 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1550,7 +1550,7 @@ impl ScheduleStage {
                         executing_queue_count = executing_queue_count.checked_add(1).unwrap();
                         to_execute_substage.send(ExecutablePayload(ee)).unwrap();
                     }
-                    debug!("schedule_once id_{:016x} [C] ch(prev: {}, exec: {}|{}), r: {}, u/c: {}/{}, (imm+provi)/max: ({}+{})/{} s: {} done: {}", random_id, (if from_disconnected { "N/A" } else { format!("{}", from_prev.len()) }), to_execute_substage.len(), from_exec.len(), runnable_queue.task_count(), address_book.uncontended_task_ids.len(), contended_count, executing_queue_count, provisioning_tracker_count, max_executing_queue_count, address_book.stuck_tasks.len(), processed_count);
+                    debug!("schedule_once id_{:016x} [C] ch(prev: {}, exec: {}|{}), r: {}, u/c: {}/{}, (imm+provi)/max: ({}+{})/{} s: {} done: {}", random_id, (if from_disconnected { "N/A" } else { &format!("{}", from_prev.len()) }), to_execute_substage.len(), from_exec.len(), runnable_queue.task_count(), address_book.uncontended_task_ids.len(), contended_count, executing_queue_count, provisioning_tracker_count, max_executing_queue_count, address_book.stuck_tasks.len(), processed_count);
                     interval_count += 1;
                     if interval_count % 100 == 0 {
                         let elapsed = last_time.elapsed();

From 95eacad4eb6056eacecd200aad0c1877776f1574 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 10 Sep 2022 21:34:16 +0900
Subject: [PATCH 2409/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 1bf2ca2acbdbce..b0e78db334ef0a 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1550,7 +1550,7 @@ impl ScheduleStage {
                         executing_queue_count = executing_queue_count.checked_add(1).unwrap();
                         to_execute_substage.send(ExecutablePayload(ee)).unwrap();
                     }
-                    debug!("schedule_once id_{:016x} [C] ch(prev: {}, exec: {}|{}), r: {}, u/c: {}/{}, (imm+provi)/max: ({}+{})/{} s: {} done: {}", random_id, (if from_disconnected { "N/A" } else { &format!("{}", from_prev.len()) }), to_execute_substage.len(), from_exec.len(), runnable_queue.task_count(), address_book.uncontended_task_ids.len(), contended_count, executing_queue_count, provisioning_tracker_count, max_executing_queue_count, address_book.stuck_tasks.len(), processed_count);
+                    debug!("schedule_once id_{:016x} [C] ch(prev: {}, exec: {}|{}), r: {}, u/c: {}/{}, (imm+provi)/max: ({}+{})/{} s: {} done: {}", random_id, (if from_disconnected { "N/A".to_str() } else { format!("{}", from_prev.len()) }), to_execute_substage.len(), from_exec.len(), runnable_queue.task_count(), address_book.uncontended_task_ids.len(), contended_count, executing_queue_count, provisioning_tracker_count, max_executing_queue_count, address_book.stuck_tasks.len(), processed_count);
                     interval_count += 1;
                     if interval_count % 100 == 0 {
                         let elapsed = last_time.elapsed();

From bb3be0210f670975b886f58e148243dd2b95fdde Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 10 Sep 2022 21:34:23 +0900
Subject: [PATCH 2410/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index b0e78db334ef0a..41cf3da1ec42ff 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1550,7 +1550,7 @@ impl ScheduleStage {
                         executing_queue_count = executing_queue_count.checked_add(1).unwrap();
                         to_execute_substage.send(ExecutablePayload(ee)).unwrap();
                     }
-                    debug!("schedule_once id_{:016x} [C] ch(prev: {}, exec: {}|{}), r: {}, u/c: {}/{}, (imm+provi)/max: ({}+{})/{} s: {} done: {}", random_id, (if from_disconnected { "N/A".to_str() } else { format!("{}", from_prev.len()) }), to_execute_substage.len(), from_exec.len(), runnable_queue.task_count(), address_book.uncontended_task_ids.len(), contended_count, executing_queue_count, provisioning_tracker_count, max_executing_queue_count, address_book.stuck_tasks.len(), processed_count);
+                    debug!("schedule_once id_{:016x} [C] ch(prev: {}, exec: {}|{}), r: {}, u/c: {}/{}, (imm+provi)/max: ({}+{})/{} s: {} done: {}", random_id, (if from_disconnected { "N/A".to_string() } else { format!("{}", from_prev.len()) }), to_execute_substage.len(), from_exec.len(), runnable_queue.task_count(), address_book.uncontended_task_ids.len(), contended_count, executing_queue_count, provisioning_tracker_count, max_executing_queue_count, address_book.stuck_tasks.len(), processed_count);
                     interval_count += 1;
                     if interval_count % 100 == 0 {
                         let elapsed = last_time.elapsed();

From a2dd7a0a53441257bb8c814c970fa8579c7042cc Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 10 Sep 2022 21:35:13 +0900
Subject: [PATCH 2411/3199] save

---
 scheduler/src/lib.rs | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 41cf3da1ec42ff..6de42f0010a40b 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1557,7 +1557,7 @@ impl ScheduleStage {
                         if elapsed > std::time::Duration::from_millis(150) {
                             let delta = (processed_count - last_processed_count) as u128;
                             let elapsed2 = elapsed.as_micros();
-                            info!("schedule_once:interval id_{:016x} ch(prev: {}, exec: {}|{}), r: {}, u/c: {}/{}, (imm+provi)/max: ({}+{})/{} s: {} done: {} ({}txs/{}us={}tps)", random_id, from_prev.len(), to_execute_substage.len(), from_exec.len(), runnable_queue.task_count(), address_book.uncontended_task_ids.len(), contended_count, executing_queue_count, provisioning_tracker_count, max_executing_queue_count, address_book.stuck_tasks.len(), processed_count, delta, elapsed.as_micros(), 1_000_000_u128*delta/elapsed2);
+                            info!("schedule_once:interval id_{:016x} ch(prev: {}, exec: {}|{}), r: {}, u/c: {}/{}, (imm+provi)/max: ({}+{})/{} s: {} done: {} ({}txs/{}us={}tps)", random_id, (if from_disconnected { "N/A".to_string() } else { format!("{}", from_prev.len()) }), to_execute_substage.len(), from_exec.len(), runnable_queue.task_count(), address_book.uncontended_task_ids.len(), contended_count, executing_queue_count, provisioning_tracker_count, max_executing_queue_count, address_book.stuck_tasks.len(), processed_count, delta, elapsed.as_micros(), 1_000_000_u128*delta/elapsed2);
                             (last_time, last_processed_count) = (std::time::Instant::now(), processed_count);
                         }
                     }
@@ -1581,9 +1581,9 @@ impl ScheduleStage {
                     if let Some(ee) = maybe_ee {
                         executing_queue_count = executing_queue_count.checked_add(1).unwrap();
                         to_execute_substage.send(ExecutablePayload(ee)).unwrap();
-                        debug!("schedule_once id_{:016x} [R] ch(prev: {}, exec: {}|{}), r: {}, u/c: {}/{}, (imm+provi)/max: ({}+{})/{} s: {} done: {}", random_id, from_prev.len(), to_execute_substage.len(), from_exec.len(), runnable_queue.task_count(), address_book.uncontended_task_ids.len(), contended_count, executing_queue_count, provisioning_tracker_count, max_executing_queue_count, address_book.stuck_tasks.len(), processed_count);
+                        debug!("schedule_once id_{:016x} [R] ch(prev: {}, exec: {}|{}), r: {}, u/c: {}/{}, (imm+provi)/max: ({}+{})/{} s: {} done: {}", random_id, (if from_disconnected { "N/A".to_string() } else { format!("{}", from_prev.len()) }), to_execute_substage.len(), from_exec.len(), runnable_queue.task_count(), address_book.uncontended_task_ids.len(), contended_count, executing_queue_count, provisioning_tracker_count, max_executing_queue_count, address_book.stuck_tasks.len(), processed_count);
                     } else {
-                        debug!("schedule_once id_{:016x} [R] ch(prev: {}, exec: {}|{}), r: {}, u/c: {}/{}, (imm+provi)/max: ({}+{})/{} s: {} done: {}", random_id, from_prev.len(), to_execute_substage.len(), from_exec.len(), runnable_queue.task_count(), address_book.uncontended_task_ids.len(), contended_count, executing_queue_count, provisioning_tracker_count, max_executing_queue_count, address_book.stuck_tasks.len(), processed_count);
+                        debug!("schedule_once id_{:016x} [R] ch(prev: {}, exec: {}|{}), r: {}, u/c: {}/{}, (imm+provi)/max: ({}+{})/{} s: {} done: {}", random_id, (if from_disconnected { "N/A".to_string() } else { format!("{}", from_prev.len()) }), to_execute_substage.len(), from_exec.len(), runnable_queue.task_count(), address_book.uncontended_task_ids.len(), contended_count, executing_queue_count, provisioning_tracker_count, max_executing_queue_count, address_book.stuck_tasks.len(), processed_count);
                         break;
                     }
                     if !from_exec.is_empty() {
@@ -1597,7 +1597,7 @@ impl ScheduleStage {
                         if elapsed > std::time::Duration::from_millis(150) {
                             let delta = (processed_count - last_processed_count) as u128;
                             let elapsed2 = elapsed.as_micros();
-                            info!("schedule_once:interval id_{:016x} ch(prev: {}, exec: {}|{}), r: {}, u/c: {}/{}, (imm+provi)/max: ({}+{})/{} s: {} done: {} ({}txs/{}us={}tps)", random_id, from_prev.len(), to_execute_substage.len(), from_exec.len(), runnable_queue.task_count(), address_book.uncontended_task_ids.len(), contended_count, executing_queue_count, provisioning_tracker_count, max_executing_queue_count, address_book.stuck_tasks.len(), processed_count, delta, elapsed.as_micros(), 1_000_000_u128*delta/elapsed2);
+                            info!("schedule_once:interval id_{:016x} ch(prev: {}, exec: {}|{}), r: {}, u/c: {}/{}, (imm+provi)/max: ({}+{})/{} s: {} done: {} ({}txs/{}us={}tps)", random_id, (if from_disconnected { "N/A".to_string() } else { format!("{}", from_prev.len()) }), to_execute_substage.len(), from_exec.len(), runnable_queue.task_count(), address_book.uncontended_task_ids.len(), contended_count, executing_queue_count, provisioning_tracker_count, max_executing_queue_count, address_book.stuck_tasks.len(), processed_count, delta, elapsed.as_micros(), 1_000_000_u128*delta/elapsed2);
                             (last_time, last_processed_count) = (std::time::Instant::now(), processed_count);
                         }
                     }

From 32d2f4608821d33131a675eb74b32c3091d3c8ac Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 10 Sep 2022 23:07:54 +0900
Subject: [PATCH 2412/3199] save

---
 runtime/src/bank.rs  |  1 +
 scheduler/src/lib.rs | 61 ++++++++++++++++++++++++--------------------
 2 files changed, 35 insertions(+), 27 deletions(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 147b9014e551fe..6f17e47bacd6db 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1307,6 +1307,7 @@ impl Default for Scheduler {
                     ee.cu = details.executed_units;
                     send_metrics.then(|| format!("{:?}", details.status))
                 } else {
+                    error!("found odd tx error: {:?}", tx_result);
                     send_metrics.then(|| format!("{:?}", tx_result))
                 };
 
diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 6de42f0010a40b..0bfd76e4922bfe 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1485,39 +1485,46 @@ impl ScheduleStage {
                 })
                 .unwrap()
         }).collect::<Vec<_>>();
-        let (mut last_time, mut last_processed_count) = (std::time::Instant::now(), 0_usize);
-        let start_time = last_time.clone();
 
+        let start_time = std::time::Instant::now();
+        let (mut last_time, mut last_processed_count) = (start_time.clone(), 0_usize);
         let (mut from_disconnected, mut from_exec_disconnected, mut no_more_work): (bool, bool, bool) = Default::default();
+
         loop {
+            let mut select_skipped = false;
+
             if !from_disconnected || executing_queue_count >= 1 {
-            crossbeam_channel::select! {
-               recv(from_exec) -> maybe_from_exec => {
-                   if let Ok(UnlockablePayload(mut processed_execution_environment)) = maybe_from_exec {
-                       executing_queue_count = executing_queue_count.checked_sub(1).unwrap();
-                       processed_count = processed_count.checked_add(1).unwrap();
-                       Self::commit_processed_execution(ast, &mut processed_execution_environment, address_book, &mut commit_clock, &mut provisioning_tracker_count);
-                       to_next_stage.send(ExaminablePayload(processed_execution_environment)).unwrap();
-                   } else {
-                       assert_eq!(from_exec.len(), 0);
-                       from_exec_disconnected |= true;
-                       info!("flushing1..: {:?} {} {} {} {}", (from_disconnected, from_exec_disconnected), runnable_queue.task_count(), contended_count, executing_queue_count, provisioning_tracker_count);
-                       if from_disconnected {
-                           break;
+                crossbeam_channel::select! {
+                   recv(from_exec) -> maybe_from_exec => {
+                       if let Ok(UnlockablePayload(mut processed_execution_environment)) = maybe_from_exec {
+                           executing_queue_count = executing_queue_count.checked_sub(1).unwrap();
+                           processed_count = processed_count.checked_add(1).unwrap();
+                           Self::commit_processed_execution(ast, &mut processed_execution_environment, address_book, &mut commit_clock, &mut provisioning_tracker_count);
+                           to_next_stage.send(ExaminablePayload(processed_execution_environment)).unwrap();
+                       } else {
+                           assert_eq!(from_exec.len(), 0);
+                           from_exec_disconnected = true;
+                           info!("flushing1..: {:?} {} {} {} {}", (from_disconnected, from_exec_disconnected), runnable_queue.task_count(), contended_count, executing_queue_count, provisioning_tracker_count);
+                           if from_disconnected {
+                               break;
+                           }
                        }
                    }
-               }
-               recv(from_prev) -> maybe_from => {
-                   if let Ok(SchedulablePayload(task)) = maybe_from {
-                       Self::register_runnable_task(task, runnable_queue, &mut sequence_time);
-                   } else {
-                       assert_eq!(from_prev.len(), 0);
-                       from_disconnected |= true;
-                       from_prev = never;
-                       trace!("flushing2..: {:?} {} {} {} {}", (from_disconnected, from_exec_disconnected), runnable_queue.task_count(), contended_count, executing_queue_count, provisioning_tracker_count);
+                   recv(from_prev) -> maybe_from => {
+                       if let Ok(SchedulablePayload(task)) = maybe_from {
+                           Self::register_runnable_task(task, runnable_queue, &mut sequence_time);
+                       } else {
+                           assert_eq!(from_prev.len(), 0);
+                           assert!(!from_disconnected);
+                           from_disconnected = true;
+                           from_prev = never;
+                           trace!("flushing2..: {:?} {} {} {} {}", (from_disconnected, from_exec_disconnected), runnable_queue.task_count(), contended_count, executing_queue_count, provisioning_tracker_count);
+                       }
                    }
-               }
-            }
+                }
+            } else {
+                // no execution at all => absolutely no active locks
+                select_skipped = true;
             }
 
            no_more_work = from_disconnected && runnable_queue.task_count() + contended_count + executing_queue_count + provisioning_tracker_count == 0;
@@ -1642,12 +1649,12 @@ impl ScheduleStage {
                     }
                 }
             }
+            assert!(!select_skipped || executing_queue_count > 0);
         }
         drop(to_next_stage);
         drop(ee_sender);
         drop(task_sender);
         drop(task_receiver);
-        info!("run finished...");
         if let Some(h) = maybe_reaper_thread_handle {
             h.join().unwrap().unwrap();
         }

From 43b835dc5852e0bcf037cc0e944830d1106be7d3 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 10 Sep 2022 23:48:41 +0900
Subject: [PATCH 2413/3199] save

---
 runtime/src/bank.rs  |  2 --
 scheduler/src/lib.rs | 25 +++++++++++++++----------
 2 files changed, 15 insertions(+), 12 deletions(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 6f17e47bacd6db..352c52d613c767 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1255,7 +1255,6 @@ impl Default for Scheduler {
         let executing_thread_handles = (0..executing_thread_count).map(|thx| {
             let (scheduled_ee_receiver, processed_ee_sender) = (scheduled_ee_receiver.clone(), processed_ee_sender.clone());
             let bank = bank.clone();
-            let mut execute_time = 0;
 
             std::thread::Builder::new().name(format!("solScExLane{:02}", thx)).spawn(move || {
             let current_thread_name = std::thread::current().name().unwrap().to_string();
@@ -1264,7 +1263,6 @@ impl Default for Scheduler {
                 let mut process_message_time = Measure::start("process_message_time");
 
                 let current_execute_clock = ee.task.execute_time();
-                execute_time += 1;
                 trace!("execute_substage: thread: {} transaction_index: {} execute_clock: {}", thx, ee.task.transaction_index_in_entries_for_replay(), current_execute_clock);
 
                 let ro_bank = bank.read().unwrap();
diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 0bfd76e4922bfe..872476f853ce4b 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1002,6 +1002,7 @@ impl ScheduleStage {
         queue_clock: &mut usize,
         provisioning_tracker_count: &mut usize,
         runnable_exclusive: bool,
+        failed_lock_count: &mut usize,
     ) -> Option<(UniqueWeight, TaskInQueue, Vec<LockAttempt>)> {
         if let Some(mut a) = address_book.fulfilled_provisional_task_ids.pop_last() {
             trace!(
@@ -1014,12 +1015,10 @@ impl ScheduleStage {
             return Some((a.0, a.1, lock_attempts));
         }
 
-        trace!("pop begin");
         loop {
             if let Some((task_source, mut next_task)) =
                 Self::select_next_task(runnable_queue, address_book, contended_count, runnable_exclusive)
             {
-                trace!("pop loop iteration");
                 let from_runnable = task_source == TaskSource::Runnable;
                 if from_runnable {
                     next_task.record_queue_time(*sequence_clock, *queue_clock);
@@ -1043,6 +1042,7 @@ impl ScheduleStage {
                     );
 
                 if unlockable_count > 0 {
+                    failed_lock_count += 1;
                     //trace!("reset_lock_for_failed_execution(): {:?} {}", (&unique_weight, from_runnable), next_task.tx.0.signature());
                     Self::reset_lock_for_failed_execution(
                         ast,
@@ -1374,6 +1374,7 @@ impl ScheduleStage {
         execute_clock: &mut usize,
         provisioning_tracker_count: &mut usize,
         runnable_exclusive: bool,
+        failed_lock_count: &mut usize,
     ) -> Option<Box<ExecutionEnvironment>> {
         let maybe_ee = Self::pop_from_queue_then_lock(
             ast,
@@ -1386,6 +1387,7 @@ impl ScheduleStage {
             queue_clock,
             provisioning_tracker_count,
             runnable_exclusive,
+            failed_lock_count,
         )
         .map(|(uw, t, ll)| {
             Self::prepare_scheduled_execution(address_book, uw, t, ll, queue_clock, execute_clock)
@@ -1416,6 +1418,8 @@ impl ScheduleStage {
         maybe_to_next_stage: Option<&crossbeam_channel::Sender<ExaminablePayload>>, // assume nonblocking
         never: &'a crossbeam_channel::Receiver<SchedulablePayload>,
     ) {
+        let start_time = std::time::Instant::now();
+        let (mut last_time, mut last_processed_count) = (start_time.clone(), 0_usize);
         let random_id = rand::thread_rng().gen::<u64>();
         info!("schedule_once:initial id_{:016x}", random_id);
 
@@ -1428,6 +1432,7 @@ impl ScheduleStage {
         let mut commit_clock = 0;
         let mut processed_count = 0_usize;
         let mut interval_count = 0;
+        let mut failed_lock_count = 0;
 
         assert!(max_executing_queue_count > 0);
 
@@ -1486,8 +1491,6 @@ impl ScheduleStage {
                 .unwrap()
         }).collect::<Vec<_>>();
 
-        let start_time = std::time::Instant::now();
-        let (mut last_time, mut last_processed_count) = (start_time.clone(), 0_usize);
         let (mut from_disconnected, mut from_exec_disconnected, mut no_more_work): (bool, bool, bool) = Default::default();
 
         loop {
@@ -1553,18 +1556,19 @@ impl ScheduleStage {
                         &mut execute_clock,
                         &mut provisioning_tracker_count,
                         false,
+                        &mut failed_lock_count,
                     ) {
                         executing_queue_count = executing_queue_count.checked_add(1).unwrap();
                         to_execute_substage.send(ExecutablePayload(ee)).unwrap();
                     }
-                    debug!("schedule_once id_{:016x} [C] ch(prev: {}, exec: {}|{}), r: {}, u/c: {}/{}, (imm+provi)/max: ({}+{})/{} s: {} done: {}", random_id, (if from_disconnected { "N/A".to_string() } else { format!("{}", from_prev.len()) }), to_execute_substage.len(), from_exec.len(), runnable_queue.task_count(), address_book.uncontended_task_ids.len(), contended_count, executing_queue_count, provisioning_tracker_count, max_executing_queue_count, address_book.stuck_tasks.len(), processed_count);
+                    debug!("schedule_once id_{:016x} [C] ch(prev: {}, exec: {}|{}), r: {}, u/c: {}/{}, (imm+provi)/max: ({}+{})/{} s: {} l(s+f): {}+{}", random_id, (if from_disconnected { "N/A".to_string() } else { format!("{}", from_prev.len()) }), to_execute_substage.len(), from_exec.len(), runnable_queue.task_count(), address_book.uncontended_task_ids.len(), contended_count, executing_queue_count, provisioning_tracker_count, max_executing_queue_count, address_book.stuck_tasks.len(), processed_count, failed_lock_count);
                     interval_count += 1;
                     if interval_count % 100 == 0 {
                         let elapsed = last_time.elapsed();
                         if elapsed > std::time::Duration::from_millis(150) {
                             let delta = (processed_count - last_processed_count) as u128;
                             let elapsed2 = elapsed.as_micros();
-                            info!("schedule_once:interval id_{:016x} ch(prev: {}, exec: {}|{}), r: {}, u/c: {}/{}, (imm+provi)/max: ({}+{})/{} s: {} done: {} ({}txs/{}us={}tps)", random_id, (if from_disconnected { "N/A".to_string() } else { format!("{}", from_prev.len()) }), to_execute_substage.len(), from_exec.len(), runnable_queue.task_count(), address_book.uncontended_task_ids.len(), contended_count, executing_queue_count, provisioning_tracker_count, max_executing_queue_count, address_book.stuck_tasks.len(), processed_count, delta, elapsed.as_micros(), 1_000_000_u128*delta/elapsed2);
+                            info!("schedule_once:interval id_{:016x} ch(prev: {}, exec: {}|{}), r: {}, u/c: {}/{}, (imm+provi)/max: ({}+{})/{} s: {} l(s+f): {}+{} ({}txs/{}us={}tps)", random_id, (if from_disconnected { "N/A".to_string() } else { format!("{}", from_prev.len()) }), to_execute_substage.len(), from_exec.len(), runnable_queue.task_count(), address_book.uncontended_task_ids.len(), contended_count, executing_queue_count, provisioning_tracker_count, max_executing_queue_count, address_book.stuck_tasks.len(), processed_count, failed_lock_count, delta, elapsed.as_micros(), 1_000_000_u128*delta/elapsed2);
                             (last_time, last_processed_count) = (std::time::Instant::now(), processed_count);
                         }
                     }
@@ -1583,14 +1587,15 @@ impl ScheduleStage {
                         &mut queue_clock,
                         &mut execute_clock,
                         &mut provisioning_tracker_count,
-                        true
+                        true,
+                        &mut failed_lock_count,
                     );
                     if let Some(ee) = maybe_ee {
                         executing_queue_count = executing_queue_count.checked_add(1).unwrap();
                         to_execute_substage.send(ExecutablePayload(ee)).unwrap();
-                        debug!("schedule_once id_{:016x} [R] ch(prev: {}, exec: {}|{}), r: {}, u/c: {}/{}, (imm+provi)/max: ({}+{})/{} s: {} done: {}", random_id, (if from_disconnected { "N/A".to_string() } else { format!("{}", from_prev.len()) }), to_execute_substage.len(), from_exec.len(), runnable_queue.task_count(), address_book.uncontended_task_ids.len(), contended_count, executing_queue_count, provisioning_tracker_count, max_executing_queue_count, address_book.stuck_tasks.len(), processed_count);
+                        debug!("schedule_once id_{:016x} [R] ch(prev: {}, exec: {}|{}), r: {}, u/c: {}/{}, (imm+provi)/max: ({}+{})/{} s: {} l(s+f): {}+{}", random_id, (if from_disconnected { "N/A".to_string() } else { format!("{}", from_prev.len()) }), to_execute_substage.len(), from_exec.len(), runnable_queue.task_count(), address_book.uncontended_task_ids.len(), contended_count, executing_queue_count, provisioning_tracker_count, max_executing_queue_count, address_book.stuck_tasks.len(), processed_count, failed_lock_count);
                     } else {
-                        debug!("schedule_once id_{:016x} [R] ch(prev: {}, exec: {}|{}), r: {}, u/c: {}/{}, (imm+provi)/max: ({}+{})/{} s: {} done: {}", random_id, (if from_disconnected { "N/A".to_string() } else { format!("{}", from_prev.len()) }), to_execute_substage.len(), from_exec.len(), runnable_queue.task_count(), address_book.uncontended_task_ids.len(), contended_count, executing_queue_count, provisioning_tracker_count, max_executing_queue_count, address_book.stuck_tasks.len(), processed_count);
+                        debug!("schedule_once id_{:016x} [R] ch(prev: {}, exec: {}|{}), r: {}, u/c: {}/{}, (imm+provi)/max: ({}+{})/{} s: {} l(s+f): {}+{}", random_id, (if from_disconnected { "N/A".to_string() } else { format!("{}", from_prev.len()) }), to_execute_substage.len(), from_exec.len(), runnable_queue.task_count(), address_book.uncontended_task_ids.len(), contended_count, executing_queue_count, provisioning_tracker_count, max_executing_queue_count, address_book.stuck_tasks.len(), processed_count, failed_lock_count);
                         break;
                     }
                     if !from_exec.is_empty() {
@@ -1604,7 +1609,7 @@ impl ScheduleStage {
                         if elapsed > std::time::Duration::from_millis(150) {
                             let delta = (processed_count - last_processed_count) as u128;
                             let elapsed2 = elapsed.as_micros();
-                            info!("schedule_once:interval id_{:016x} ch(prev: {}, exec: {}|{}), r: {}, u/c: {}/{}, (imm+provi)/max: ({}+{})/{} s: {} done: {} ({}txs/{}us={}tps)", random_id, (if from_disconnected { "N/A".to_string() } else { format!("{}", from_prev.len()) }), to_execute_substage.len(), from_exec.len(), runnable_queue.task_count(), address_book.uncontended_task_ids.len(), contended_count, executing_queue_count, provisioning_tracker_count, max_executing_queue_count, address_book.stuck_tasks.len(), processed_count, delta, elapsed.as_micros(), 1_000_000_u128*delta/elapsed2);
+                            info!("schedule_once:interval id_{:016x} ch(prev: {}, exec: {}|{}), r: {}, u/c: {}/{}, (imm+provi)/max: ({}+{})/{} s: {} l(s+f): {}+{} ({}txs/{}us={}tps)", random_id, (if from_disconnected { "N/A".to_string() } else { format!("{}", from_prev.len()) }), to_execute_substage.len(), from_exec.len(), runnable_queue.task_count(), address_book.uncontended_task_ids.len(), contended_count, executing_queue_count, provisioning_tracker_count, max_executing_queue_count, address_book.stuck_tasks.len(), processed_count, failed_lock_count, delta, elapsed.as_micros(), 1_000_000_u128*delta/elapsed2);
                             (last_time, last_processed_count) = (std::time::Instant::now(), processed_count);
                         }
                     }

From 43f16ce4da44ff7ff0080cf15cbb08e4bbc843c7 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 10 Sep 2022 23:49:29 +0900
Subject: [PATCH 2414/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 872476f853ce4b..819ed4f9dbf047 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1002,7 +1002,7 @@ impl ScheduleStage {
         queue_clock: &mut usize,
         provisioning_tracker_count: &mut usize,
         runnable_exclusive: bool,
-        failed_lock_count: &mut usize,
+        &failed_lock_count: &mut usize,
     ) -> Option<(UniqueWeight, TaskInQueue, Vec<LockAttempt>)> {
         if let Some(mut a) = address_book.fulfilled_provisional_task_ids.pop_last() {
             trace!(

From 6cd045e83a1a2420000dd124eee39904d8449212 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 10 Sep 2022 23:49:44 +0900
Subject: [PATCH 2415/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 819ed4f9dbf047..3c02dc2a76f4e7 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1002,7 +1002,7 @@ impl ScheduleStage {
         queue_clock: &mut usize,
         provisioning_tracker_count: &mut usize,
         runnable_exclusive: bool,
-        &failed_lock_count: &mut usize,
+        &mut failed_lock_count: &mut usize,
     ) -> Option<(UniqueWeight, TaskInQueue, Vec<LockAttempt>)> {
         if let Some(mut a) = address_book.fulfilled_provisional_task_ids.pop_last() {
             trace!(

From e91af2cb0c60efa0c533733e3ad1f928ff35a265 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 10 Sep 2022 23:49:54 +0900
Subject: [PATCH 2416/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 3c02dc2a76f4e7..9c3a5feaded9af 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1002,7 +1002,7 @@ impl ScheduleStage {
         queue_clock: &mut usize,
         provisioning_tracker_count: &mut usize,
         runnable_exclusive: bool,
-        &mut failed_lock_count: &mut usize,
+        &mut failed_lock_count: usize,
     ) -> Option<(UniqueWeight, TaskInQueue, Vec<LockAttempt>)> {
         if let Some(mut a) = address_book.fulfilled_provisional_task_ids.pop_last() {
             trace!(

From 49c5fe51dc3b71b086572ee8e8d31e9ca55a1df0 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 10 Sep 2022 23:50:22 +0900
Subject: [PATCH 2417/3199] save

---
 scheduler/src/lib.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 9c3a5feaded9af..1048ced1976181 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1002,7 +1002,7 @@ impl ScheduleStage {
         queue_clock: &mut usize,
         provisioning_tracker_count: &mut usize,
         runnable_exclusive: bool,
-        &mut failed_lock_count: usize,
+        failed_lock_count: &mut usize,
     ) -> Option<(UniqueWeight, TaskInQueue, Vec<LockAttempt>)> {
         if let Some(mut a) = address_book.fulfilled_provisional_task_ids.pop_last() {
             trace!(
@@ -1042,7 +1042,7 @@ impl ScheduleStage {
                     );
 
                 if unlockable_count > 0 {
-                    failed_lock_count += 1;
+                    *failed_lock_count += 1;
                     //trace!("reset_lock_for_failed_execution(): {:?} {}", (&unique_weight, from_runnable), next_task.tx.0.signature());
                     Self::reset_lock_for_failed_execution(
                         ast,

From 957527614ed09acf2a386c4c795e505943ba71cb Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 10 Sep 2022 23:51:05 +0900
Subject: [PATCH 2418/3199] save

---
 scheduler/src/lib.rs | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 1048ced1976181..1a49d999cd6df3 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1561,14 +1561,14 @@ impl ScheduleStage {
                         executing_queue_count = executing_queue_count.checked_add(1).unwrap();
                         to_execute_substage.send(ExecutablePayload(ee)).unwrap();
                     }
-                    debug!("schedule_once id_{:016x} [C] ch(prev: {}, exec: {}|{}), r: {}, u/c: {}/{}, (imm+provi)/max: ({}+{})/{} s: {} l(s+f): {}+{}", random_id, (if from_disconnected { "N/A".to_string() } else { format!("{}", from_prev.len()) }), to_execute_substage.len(), from_exec.len(), runnable_queue.task_count(), address_book.uncontended_task_ids.len(), contended_count, executing_queue_count, provisioning_tracker_count, max_executing_queue_count, address_book.stuck_tasks.len(), processed_count, failed_lock_count);
+                    debug!("schedule_once id_{:016x} [C] ch(prev: {}, exec: {}|{}), r: {}, u/c: {}/{}, (imm+provi)/max: ({}+{})/{} s: {} l(s+f): {}+{}", random_id, (if from_disconnected { "-".to_string() } else { format!("{}", from_prev.len()) }), to_execute_substage.len(), from_exec.len(), runnable_queue.task_count(), address_book.uncontended_task_ids.len(), contended_count, executing_queue_count, provisioning_tracker_count, max_executing_queue_count, address_book.stuck_tasks.len(), processed_count, failed_lock_count);
                     interval_count += 1;
                     if interval_count % 100 == 0 {
                         let elapsed = last_time.elapsed();
                         if elapsed > std::time::Duration::from_millis(150) {
                             let delta = (processed_count - last_processed_count) as u128;
                             let elapsed2 = elapsed.as_micros();
-                            info!("schedule_once:interval id_{:016x} ch(prev: {}, exec: {}|{}), r: {}, u/c: {}/{}, (imm+provi)/max: ({}+{})/{} s: {} l(s+f): {}+{} ({}txs/{}us={}tps)", random_id, (if from_disconnected { "N/A".to_string() } else { format!("{}", from_prev.len()) }), to_execute_substage.len(), from_exec.len(), runnable_queue.task_count(), address_book.uncontended_task_ids.len(), contended_count, executing_queue_count, provisioning_tracker_count, max_executing_queue_count, address_book.stuck_tasks.len(), processed_count, failed_lock_count, delta, elapsed.as_micros(), 1_000_000_u128*delta/elapsed2);
+                            info!("schedule_once:interval id_{:016x} ch(prev: {}, exec: {}|{}), r: {}, u/c: {}/{}, (imm+provi)/max: ({}+{})/{} s: {} l(s+f): {}+{} ({}txs/{}us={}tps)", random_id, (if from_disconnected { "-".to_string() } else { format!("{}", from_prev.len()) }), to_execute_substage.len(), from_exec.len(), runnable_queue.task_count(), address_book.uncontended_task_ids.len(), contended_count, executing_queue_count, provisioning_tracker_count, max_executing_queue_count, address_book.stuck_tasks.len(), processed_count, failed_lock_count, delta, elapsed.as_micros(), 1_000_000_u128*delta/elapsed2);
                             (last_time, last_processed_count) = (std::time::Instant::now(), processed_count);
                         }
                     }
@@ -1593,9 +1593,9 @@ impl ScheduleStage {
                     if let Some(ee) = maybe_ee {
                         executing_queue_count = executing_queue_count.checked_add(1).unwrap();
                         to_execute_substage.send(ExecutablePayload(ee)).unwrap();
-                        debug!("schedule_once id_{:016x} [R] ch(prev: {}, exec: {}|{}), r: {}, u/c: {}/{}, (imm+provi)/max: ({}+{})/{} s: {} l(s+f): {}+{}", random_id, (if from_disconnected { "N/A".to_string() } else { format!("{}", from_prev.len()) }), to_execute_substage.len(), from_exec.len(), runnable_queue.task_count(), address_book.uncontended_task_ids.len(), contended_count, executing_queue_count, provisioning_tracker_count, max_executing_queue_count, address_book.stuck_tasks.len(), processed_count, failed_lock_count);
+                        debug!("schedule_once id_{:016x} [R] ch(prev: {}, exec: {}|{}), r: {}, u/c: {}/{}, (imm+provi)/max: ({}+{})/{} s: {} l(s+f): {}+{}", random_id, (if from_disconnected { "-".to_string() } else { format!("{}", from_prev.len()) }), to_execute_substage.len(), from_exec.len(), runnable_queue.task_count(), address_book.uncontended_task_ids.len(), contended_count, executing_queue_count, provisioning_tracker_count, max_executing_queue_count, address_book.stuck_tasks.len(), processed_count, failed_lock_count);
                     } else {
-                        debug!("schedule_once id_{:016x} [R] ch(prev: {}, exec: {}|{}), r: {}, u/c: {}/{}, (imm+provi)/max: ({}+{})/{} s: {} l(s+f): {}+{}", random_id, (if from_disconnected { "N/A".to_string() } else { format!("{}", from_prev.len()) }), to_execute_substage.len(), from_exec.len(), runnable_queue.task_count(), address_book.uncontended_task_ids.len(), contended_count, executing_queue_count, provisioning_tracker_count, max_executing_queue_count, address_book.stuck_tasks.len(), processed_count, failed_lock_count);
+                        debug!("schedule_once id_{:016x} [R] ch(prev: {}, exec: {}|{}), r: {}, u/c: {}/{}, (imm+provi)/max: ({}+{})/{} s: {} l(s+f): {}+{}", random_id, (if from_disconnected { "-".to_string() } else { format!("{}", from_prev.len()) }), to_execute_substage.len(), from_exec.len(), runnable_queue.task_count(), address_book.uncontended_task_ids.len(), contended_count, executing_queue_count, provisioning_tracker_count, max_executing_queue_count, address_book.stuck_tasks.len(), processed_count, failed_lock_count);
                         break;
                     }
                     if !from_exec.is_empty() {
@@ -1609,7 +1609,7 @@ impl ScheduleStage {
                         if elapsed > std::time::Duration::from_millis(150) {
                             let delta = (processed_count - last_processed_count) as u128;
                             let elapsed2 = elapsed.as_micros();
-                            info!("schedule_once:interval id_{:016x} ch(prev: {}, exec: {}|{}), r: {}, u/c: {}/{}, (imm+provi)/max: ({}+{})/{} s: {} l(s+f): {}+{} ({}txs/{}us={}tps)", random_id, (if from_disconnected { "N/A".to_string() } else { format!("{}", from_prev.len()) }), to_execute_substage.len(), from_exec.len(), runnable_queue.task_count(), address_book.uncontended_task_ids.len(), contended_count, executing_queue_count, provisioning_tracker_count, max_executing_queue_count, address_book.stuck_tasks.len(), processed_count, failed_lock_count, delta, elapsed.as_micros(), 1_000_000_u128*delta/elapsed2);
+                            info!("schedule_once:interval id_{:016x} ch(prev: {}, exec: {}|{}), r: {}, u/c: {}/{}, (imm+provi)/max: ({}+{})/{} s: {} l(s+f): {}+{} ({}txs/{}us={}tps)", random_id, (if from_disconnected { "-".to_string() } else { format!("{}", from_prev.len()) }), to_execute_substage.len(), from_exec.len(), runnable_queue.task_count(), address_book.uncontended_task_ids.len(), contended_count, executing_queue_count, provisioning_tracker_count, max_executing_queue_count, address_book.stuck_tasks.len(), processed_count, failed_lock_count, delta, elapsed.as_micros(), 1_000_000_u128*delta/elapsed2);
                             (last_time, last_processed_count) = (std::time::Instant::now(), processed_count);
                         }
                     }

From 3a32fa2c92cddec1c11eee9c29854a3f3a5bbad4 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 10 Sep 2022 23:53:45 +0900
Subject: [PATCH 2419/3199] save

---
 scheduler/src/lib.rs | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 1a49d999cd6df3..c7aa16c6a91fce 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1538,6 +1538,17 @@ impl ScheduleStage {
             let mut first_iteration = true;
             let (mut empty_from, mut empty_from_exec) = (false, false);
             let (mut from_len, mut from_exec_len) = (0, 0);
+            let log =  || {
+                    if interval_count % 100 == 0 {
+                        let elapsed = last_time.elapsed();
+                        if elapsed > std::time::Duration::from_millis(150) {
+                            let delta = (processed_count - last_processed_count) as u128;
+                            let elapsed2 = elapsed.as_micros();
+                            info!("schedule_once:interval id_{:016x} ch(prev: {}, exec: {}|{}), r: {}, u/c: {}/{}, (imm+provi)/max: ({}+{})/{} s: {} l(s+f): {}+{} ({}txs/{}us={}tps)", random_id, (if from_disconnected { "-".to_string() } else { format!("{}", from_prev.len()) }), to_execute_substage.len(), from_exec.len(), runnable_queue.task_count(), address_book.uncontended_task_ids.len(), contended_count, executing_queue_count, provisioning_tracker_count, max_executing_queue_count, address_book.stuck_tasks.len(), processed_count, failed_lock_count, delta, elapsed.as_micros(), 1_000_000_u128*delta/elapsed2);
+                            (last_time, last_processed_count) = (std::time::Instant::now(), processed_count);
+                        }
+                    }
+            }
 
             loop {
                 let executing_like_count = executing_queue_count + provisioning_tracker_count;

From a46904a90aa59b610810c5b148034cbb5658581e Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 10 Sep 2022 23:53:50 +0900
Subject: [PATCH 2420/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index c7aa16c6a91fce..6131b00d99a9a4 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1548,7 +1548,7 @@ impl ScheduleStage {
                             (last_time, last_processed_count) = (std::time::Instant::now(), processed_count);
                         }
                     }
-            }
+            };
 
             loop {
                 let executing_like_count = executing_queue_count + provisioning_tracker_count;

From a11092d882408c0f74090cdc913c07c993273645 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 10 Sep 2022 23:54:02 +0900
Subject: [PATCH 2421/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 6131b00d99a9a4..fdc9d0c4f8aaba 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1539,7 +1539,7 @@ impl ScheduleStage {
             let (mut empty_from, mut empty_from_exec) = (false, false);
             let (mut from_len, mut from_exec_len) = (0, 0);
             let log =  || {
-                    if interval_count % 100 == 0 {
+                    if intrval_count % 100 == 0 {
                         let elapsed = last_time.elapsed();
                         if elapsed > std::time::Duration::from_millis(150) {
                             let delta = (processed_count - last_processed_count) as u128;

From 3572319f378f41a4ffdd339ba068c64d200f8b8f Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 10 Sep 2022 23:54:58 +0900
Subject: [PATCH 2422/3199] save

---
 scheduler/src/lib.rs | 30 +++++++++++-------------------
 1 file changed, 11 insertions(+), 19 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index fdc9d0c4f8aaba..3164aa6032e52f 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1538,16 +1538,17 @@ impl ScheduleStage {
             let mut first_iteration = true;
             let (mut empty_from, mut empty_from_exec) = (false, false);
             let (mut from_len, mut from_exec_len) = (0, 0);
-            let log =  || {
-                    if intrval_count % 100 == 0 {
-                        let elapsed = last_time.elapsed();
-                        if elapsed > std::time::Duration::from_millis(150) {
-                            let delta = (processed_count - last_processed_count) as u128;
-                            let elapsed2 = elapsed.as_micros();
-                            info!("schedule_once:interval id_{:016x} ch(prev: {}, exec: {}|{}), r: {}, u/c: {}/{}, (imm+provi)/max: ({}+{})/{} s: {} l(s+f): {}+{} ({}txs/{}us={}tps)", random_id, (if from_disconnected { "-".to_string() } else { format!("{}", from_prev.len()) }), to_execute_substage.len(), from_exec.len(), runnable_queue.task_count(), address_book.uncontended_task_ids.len(), contended_count, executing_queue_count, provisioning_tracker_count, max_executing_queue_count, address_book.stuck_tasks.len(), processed_count, failed_lock_count, delta, elapsed.as_micros(), 1_000_000_u128*delta/elapsed2);
-                            (last_time, last_processed_count) = (std::time::Instant::now(), processed_count);
-                        }
+            let log_at_intervals = || {
+                interval_count += 1;
+                if interval_count % 100 == 0 {
+                    let elapsed = last_time.elapsed();
+                    if elapsed > std::time::Duration::from_millis(150) {
+                        let delta = (processed_count - last_processed_count) as u128;
+                        let elapsed2 = elapsed.as_micros();
+                        info!("schedule_once:interval id_{:016x} ch(prev: {}, exec: {}|{}), r: {}, u/c: {}/{}, (imm+provi)/max: ({}+{})/{} s: {} l(s+f): {}+{} ({}txs/{}us={}tps)", random_id, (if from_disconnected { "-".to_string() } else { format!("{}", from_prev.len()) }), to_execute_substage.len(), from_exec.len(), runnable_queue.task_count(), address_book.uncontended_task_ids.len(), contended_count, executing_queue_count, provisioning_tracker_count, max_executing_queue_count, address_book.stuck_tasks.len(), processed_count, failed_lock_count, delta, elapsed.as_micros(), 1_000_000_u128*delta/elapsed2);
+                        (last_time, last_processed_count) = (std::time::Instant::now(), processed_count);
                     }
+                }
             };
 
             loop {
@@ -1573,16 +1574,7 @@ impl ScheduleStage {
                         to_execute_substage.send(ExecutablePayload(ee)).unwrap();
                     }
                     debug!("schedule_once id_{:016x} [C] ch(prev: {}, exec: {}|{}), r: {}, u/c: {}/{}, (imm+provi)/max: ({}+{})/{} s: {} l(s+f): {}+{}", random_id, (if from_disconnected { "-".to_string() } else { format!("{}", from_prev.len()) }), to_execute_substage.len(), from_exec.len(), runnable_queue.task_count(), address_book.uncontended_task_ids.len(), contended_count, executing_queue_count, provisioning_tracker_count, max_executing_queue_count, address_book.stuck_tasks.len(), processed_count, failed_lock_count);
-                    interval_count += 1;
-                    if interval_count % 100 == 0 {
-                        let elapsed = last_time.elapsed();
-                        if elapsed > std::time::Duration::from_millis(150) {
-                            let delta = (processed_count - last_processed_count) as u128;
-                            let elapsed2 = elapsed.as_micros();
-                            info!("schedule_once:interval id_{:016x} ch(prev: {}, exec: {}|{}), r: {}, u/c: {}/{}, (imm+provi)/max: ({}+{})/{} s: {} l(s+f): {}+{} ({}txs/{}us={}tps)", random_id, (if from_disconnected { "-".to_string() } else { format!("{}", from_prev.len()) }), to_execute_substage.len(), from_exec.len(), runnable_queue.task_count(), address_book.uncontended_task_ids.len(), contended_count, executing_queue_count, provisioning_tracker_count, max_executing_queue_count, address_book.stuck_tasks.len(), processed_count, failed_lock_count, delta, elapsed.as_micros(), 1_000_000_u128*delta/elapsed2);
-                            (last_time, last_processed_count) = (std::time::Instant::now(), processed_count);
-                        }
-                    }
+                    log_at_intervals();
                 }
                 while executing_queue_count + provisioning_tracker_count < max_executing_queue_count {
                     let prefer_immediate = true; //provisioning_tracker_count / 4 > executing_queue_count;

From 73b55686a04ab1301f7ad8969d52f961d02aebd2 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 10 Sep 2022 23:59:28 +0900
Subject: [PATCH 2423/3199] save

---
 scheduler/src/lib.rs | 30 ++++++++++++++----------------
 1 file changed, 14 insertions(+), 16 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 3164aa6032e52f..71a8b18e64e161 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -874,12 +874,13 @@ enum TaskSource {
     Stuck,
 }
 
-/*
 enum SelectionContext {
-    Runnable,
-    Contended(u8, Vec<TaskInQueue>),
+    OnlyRunnable,
+    OnlyContended(usize),
 }
 
+/*
+
 impl SelectionContext {
     fn should_continue(&self) -> bool {
         match(self) {
@@ -1538,18 +1539,6 @@ impl ScheduleStage {
             let mut first_iteration = true;
             let (mut empty_from, mut empty_from_exec) = (false, false);
             let (mut from_len, mut from_exec_len) = (0, 0);
-            let log_at_intervals = || {
-                interval_count += 1;
-                if interval_count % 100 == 0 {
-                    let elapsed = last_time.elapsed();
-                    if elapsed > std::time::Duration::from_millis(150) {
-                        let delta = (processed_count - last_processed_count) as u128;
-                        let elapsed2 = elapsed.as_micros();
-                        info!("schedule_once:interval id_{:016x} ch(prev: {}, exec: {}|{}), r: {}, u/c: {}/{}, (imm+provi)/max: ({}+{})/{} s: {} l(s+f): {}+{} ({}txs/{}us={}tps)", random_id, (if from_disconnected { "-".to_string() } else { format!("{}", from_prev.len()) }), to_execute_substage.len(), from_exec.len(), runnable_queue.task_count(), address_book.uncontended_task_ids.len(), contended_count, executing_queue_count, provisioning_tracker_count, max_executing_queue_count, address_book.stuck_tasks.len(), processed_count, failed_lock_count, delta, elapsed.as_micros(), 1_000_000_u128*delta/elapsed2);
-                        (last_time, last_processed_count) = (std::time::Instant::now(), processed_count);
-                    }
-                }
-            };
 
             loop {
                 let executing_like_count = executing_queue_count + provisioning_tracker_count;
@@ -1574,7 +1563,16 @@ impl ScheduleStage {
                         to_execute_substage.send(ExecutablePayload(ee)).unwrap();
                     }
                     debug!("schedule_once id_{:016x} [C] ch(prev: {}, exec: {}|{}), r: {}, u/c: {}/{}, (imm+provi)/max: ({}+{})/{} s: {} l(s+f): {}+{}", random_id, (if from_disconnected { "-".to_string() } else { format!("{}", from_prev.len()) }), to_execute_substage.len(), from_exec.len(), runnable_queue.task_count(), address_book.uncontended_task_ids.len(), contended_count, executing_queue_count, provisioning_tracker_count, max_executing_queue_count, address_book.stuck_tasks.len(), processed_count, failed_lock_count);
-                    log_at_intervals();
+                    interval_count += 1;
+                    if interval_count % 100 == 0 {
+                        let elapsed = last_time.elapsed();
+                        if elapsed > std::time::Duration::from_millis(150) {
+                            let delta = (processed_count - last_processed_count) as u128;
+                            let elapsed2 = elapsed.as_micros();
+                            info!("schedule_once:interval id_{:016x} ch(prev: {}, exec: {}|{}), r: {}, u/c: {}/{}, (imm+provi)/max: ({}+{})/{} s: {} l(s+f): {}+{} ({}txs/{}us={}tps)", random_id, (if from_disconnected { "-".to_string() } else { format!("{}", from_prev.len()) }), to_execute_substage.len(), from_exec.len(), runnable_queue.task_count(), address_book.uncontended_task_ids.len(), contended_count, executing_queue_count, provisioning_tracker_count, max_executing_queue_count, address_book.stuck_tasks.len(), processed_count, failed_lock_count, delta, elapsed.as_micros(), 1_000_000_u128*delta/elapsed2);
+                            (last_time, last_processed_count) = (std::time::Instant::now(), processed_count);
+                        }
+                    }
                 }
                 while executing_queue_count + provisioning_tracker_count < max_executing_queue_count {
                     let prefer_immediate = true; //provisioning_tracker_count / 4 > executing_queue_count;

From b455ec801fcf5ae84c3c31459138be201ea1bfdd Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 11 Sep 2022 00:00:06 +0900
Subject: [PATCH 2424/3199] save

---
 scheduler/src/lib.rs | 10 ++++------
 1 file changed, 4 insertions(+), 6 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 71a8b18e64e161..139189abaafce4 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -879,24 +879,22 @@ enum SelectionContext {
     OnlyContended(usize),
 }
 
-/*
 
 impl SelectionContext {
     fn should_continue(&self) -> bool {
         match(self) {
-            SelectionContext::Runnable => true,
-            SelectionContext::Contended(failure_count) => failure_count < 2,
+            SelectionContext::Runnable -> true,
+            SelectionContext::Contended(failure_count) -> failure_count < 2,
         }
     }
 
     fn runnable_exclusive(&self) -> bool {
         match(self) {
-            SelectionContext::Runnable => true,
-            SelectionContext::Contended(_) => false,
+            SelectionContext::Runnable -> true,
+            SelectionContext::Contended(_) -> false,
         }
     }
 }
-*/
 
 impl ScheduleStage {
     fn push_to_runnable_queue(task: TaskInQueue, runnable_queue: &mut TaskQueue) {

From 5cd55b3985f5e6c2668dda374effad17b6ef43bb Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 11 Sep 2022 00:00:23 +0900
Subject: [PATCH 2425/3199] save

---
 scheduler/src/lib.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 139189abaafce4..9e0c3e8d9089dc 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -882,14 +882,14 @@ enum SelectionContext {
 
 impl SelectionContext {
     fn should_continue(&self) -> bool {
-        match(self) {
+        match self {
             SelectionContext::Runnable -> true,
             SelectionContext::Contended(failure_count) -> failure_count < 2,
         }
     }
 
     fn runnable_exclusive(&self) -> bool {
-        match(self) {
+        match self {
             SelectionContext::Runnable -> true,
             SelectionContext::Contended(_) -> false,
         }

From 8c693fdbe1b0db35bc385484fa8df3920d4fa511 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 11 Sep 2022 00:00:38 +0900
Subject: [PATCH 2426/3199] save

---
 scheduler/src/lib.rs | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 9e0c3e8d9089dc..ccf01bb6118d3e 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -883,15 +883,15 @@ enum SelectionContext {
 impl SelectionContext {
     fn should_continue(&self) -> bool {
         match self {
-            SelectionContext::Runnable -> true,
-            SelectionContext::Contended(failure_count) -> failure_count < 2,
+            SelectionContext::OnlyRunnable -> true,
+            SelectionContext::OnlyContended(failure_count) -> failure_count < 2,
         }
     }
 
     fn runnable_exclusive(&self) -> bool {
         match self {
-            SelectionContext::Runnable -> true,
-            SelectionContext::Contended(_) -> false,
+            SelectionContext::OnlyRunnable -> true,
+            SelectionContext::OnlyContended(_) -> false,
         }
     }
 }

From 8271fba918d2e14fe68524550b862e9adca4279c Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 11 Sep 2022 00:00:51 +0900
Subject: [PATCH 2427/3199] save

---
 scheduler/src/lib.rs | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index ccf01bb6118d3e..91c43952fe8195 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -883,15 +883,15 @@ enum SelectionContext {
 impl SelectionContext {
     fn should_continue(&self) -> bool {
         match self {
-            SelectionContext::OnlyRunnable -> true,
-            SelectionContext::OnlyContended(failure_count) -> failure_count < 2,
+            SelectionContext::OnlyRunnable => true,
+            SelectionContext::OnlyContended(failure_count) => failure_count < 2,
         }
     }
 
     fn runnable_exclusive(&self) -> bool {
         match self {
-            SelectionContext::OnlyRunnable -> true,
-            SelectionContext::OnlyContended(_) -> false,
+            SelectionContext::OnlyRunnable => true,
+            SelectionContext::OnlyContended(_) => false,
         }
     }
 }

From ea9305216614202f0c22d1e92c19cda35572fcbc Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 11 Sep 2022 00:00:59 +0900
Subject: [PATCH 2428/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 91c43952fe8195..106faf7ec04e06 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -884,7 +884,7 @@ impl SelectionContext {
     fn should_continue(&self) -> bool {
         match self {
             SelectionContext::OnlyRunnable => true,
-            SelectionContext::OnlyContended(failure_count) => failure_count < 2,
+            SelectionContext::OnlyContended(ref failure_count) => failure_count < 2,
         }
     }
 

From 058d70f6515a5d371be2fb619ca01956322dc134 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 11 Sep 2022 00:01:05 +0900
Subject: [PATCH 2429/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 106faf7ec04e06..1b7c7466420db7 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -884,7 +884,7 @@ impl SelectionContext {
     fn should_continue(&self) -> bool {
         match self {
             SelectionContext::OnlyRunnable => true,
-            SelectionContext::OnlyContended(ref failure_count) => failure_count < 2,
+            SelectionContext::OnlyContended(&failure_count) => failure_count < 2,
         }
     }
 

From bba13dbbad8ceff52a0291a020a3263d89615556 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 11 Sep 2022 00:01:16 +0900
Subject: [PATCH 2430/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 1b7c7466420db7..7bdc30a2934567 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -884,7 +884,7 @@ impl SelectionContext {
     fn should_continue(&self) -> bool {
         match self {
             SelectionContext::OnlyRunnable => true,
-            SelectionContext::OnlyContended(&failure_count) => failure_count < 2,
+            SelectionContext::OnlyContended(failure_count) => *failure_count < 2,
         }
     }
 

From 818297e15213ae1218eb60336f2c170152900177 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 11 Sep 2022 00:09:31 +0900
Subject: [PATCH 2431/3199] save

---
 scheduler/src/lib.rs | 38 ++++++++++++++++++++------------------
 1 file changed, 20 insertions(+), 18 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 7bdc30a2934567..2fc68ac960d548 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -874,24 +874,24 @@ enum TaskSource {
     Stuck,
 }
 
-enum SelectionContext {
-    OnlyRunnable,
-    OnlyContended(usize),
+enum TaskSelection {
+    OnlyFromRunnable,
+    OnlyFromContended(usize),
 }
 
 
-impl SelectionContext {
+impl TaskSelection {
     fn should_continue(&self) -> bool {
         match self {
-            SelectionContext::OnlyRunnable => true,
-            SelectionContext::OnlyContended(failure_count) => *failure_count < 2,
+            TaskSelection::OnlyFromRunnable => true,
+            TaskSelection::OnlyFromContended(failure_count) => *failure_count < 2,
         }
     }
 
     fn runnable_exclusive(&self) -> bool {
         match self {
-            SelectionContext::OnlyRunnable => true,
-            SelectionContext::OnlyContended(_) => false,
+            TaskSelection::OnlyFromRunnable => true,
+            TaskSelection::OnlyFromContended(_) => false,
         }
     }
 }
@@ -913,7 +913,7 @@ impl ScheduleStage {
         runnable_queue: &'a mut TaskQueue,
         address_book: &mut AddressBook,
         contended_count: &usize,
-        runnable_exclusive: bool,
+        task_selection: &mut TaskSelection,
     ) -> Option<(TaskSource, TaskInQueue)> {
         match (
             runnable_queue.heaviest_entry_to_execute(),
@@ -921,7 +921,7 @@ impl ScheduleStage {
         ) {
             (Some(heaviest_runnable_entry), None) => {
                 trace!("select: runnable only");
-                if runnable_exclusive {
+                if task_selection.runnable_exclusive() {
                     let t = heaviest_runnable_entry.remove();
                     Some((TaskSource::Runnable, t))
                 } else {
@@ -930,7 +930,7 @@ impl ScheduleStage {
             }
             (None, Some(weight_from_contended)) => {
                 trace!("select: contended only");
-                if runnable_exclusive {
+                if task_selection.runnable_exclusive() {
                     None
                 } else {
                     let t = weight_from_contended.remove();
@@ -950,7 +950,7 @@ impl ScheduleStage {
                     Some((TaskSource::Runnable, t))
                     */
                 } else if uw > weight_from_runnable {
-                    if runnable_exclusive {
+                    if task_selection.runnable_exclusive() {
                         trace!("select: contended > runnnable, runnable_exclusive");
                         let t = heaviest_runnable_entry.remove();
                         Some((TaskSource::Runnable, t))
@@ -1000,7 +1000,7 @@ impl ScheduleStage {
         sequence_clock: &usize,
         queue_clock: &mut usize,
         provisioning_tracker_count: &mut usize,
-        runnable_exclusive: bool,
+        task_selection: &mut TaskSelection,
         failed_lock_count: &mut usize,
     ) -> Option<(UniqueWeight, TaskInQueue, Vec<LockAttempt>)> {
         if let Some(mut a) = address_book.fulfilled_provisional_task_ids.pop_last() {
@@ -1016,7 +1016,7 @@ impl ScheduleStage {
 
         loop {
             if let Some((task_source, mut next_task)) =
-                Self::select_next_task(runnable_queue, address_book, contended_count, runnable_exclusive)
+                Self::select_next_task(runnable_queue, address_book, contended_count, task_selection)
             {
                 let from_runnable = task_source == TaskSource::Runnable;
                 if from_runnable {
@@ -1372,7 +1372,7 @@ impl ScheduleStage {
         queue_clock: &mut usize,
         execute_clock: &mut usize,
         provisioning_tracker_count: &mut usize,
-        runnable_exclusive: bool,
+        task_selection: &mut TaskSelection,
         failed_lock_count: &mut usize,
     ) -> Option<Box<ExecutionEnvironment>> {
         let maybe_ee = Self::pop_from_queue_then_lock(
@@ -1385,7 +1385,7 @@ impl ScheduleStage {
             sequence_time,
             queue_clock,
             provisioning_tracker_count,
-            runnable_exclusive,
+            task_selection,
             failed_lock_count,
         )
         .map(|(uw, t, ll)| {
@@ -1542,6 +1542,7 @@ impl ScheduleStage {
                 let executing_like_count = executing_queue_count + provisioning_tracker_count;
                 if executing_like_count < max_executing_queue_count {
                     let prefer_immediate = true; //provisioning_tracker_count / 4 > executing_queue_count;
+                    let mut selection = TaskSelection::OnlyFromContended(0);
 
                     if let Some(ee) = Self::schedule_next_execution(
                         ast,
@@ -1554,7 +1555,7 @@ impl ScheduleStage {
                         &mut queue_clock,
                         &mut execute_clock,
                         &mut provisioning_tracker_count,
-                        false,
+                        &mut selection,
                         &mut failed_lock_count,
                     ) {
                         executing_queue_count = executing_queue_count.checked_add(1).unwrap();
@@ -1574,6 +1575,7 @@ impl ScheduleStage {
                 }
                 while executing_queue_count + provisioning_tracker_count < max_executing_queue_count {
                     let prefer_immediate = true; //provisioning_tracker_count / 4 > executing_queue_count;
+                    let mut selection = TaskSelection::OnlyFromRunnable();
 
                     let maybe_ee = Self::schedule_next_execution(
                         ast,
@@ -1586,7 +1588,7 @@ impl ScheduleStage {
                         &mut queue_clock,
                         &mut execute_clock,
                         &mut provisioning_tracker_count,
-                        true,
+                        &mut selection,
                         &mut failed_lock_count,
                     );
                     if let Some(ee) = maybe_ee {

From d3cad56af79f994cf765c618a401c5e8dde8ed93 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 11 Sep 2022 00:09:43 +0900
Subject: [PATCH 2432/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 2fc68ac960d548..accdb4c3a55650 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1575,7 +1575,7 @@ impl ScheduleStage {
                 }
                 while executing_queue_count + provisioning_tracker_count < max_executing_queue_count {
                     let prefer_immediate = true; //provisioning_tracker_count / 4 > executing_queue_count;
-                    let mut selection = TaskSelection::OnlyFromRunnable();
+                    let mut selection = TaskSelection::OnlyFromRunnable;
 
                     let maybe_ee = Self::schedule_next_execution(
                         ast,

From 21be88b04370a226de4e2a677d967f1dbc291744 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 11 Sep 2022 00:11:25 +0900
Subject: [PATCH 2433/3199] save

---
 scheduler/src/lib.rs | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index accdb4c3a55650..5d22719a218d66 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -881,7 +881,7 @@ enum TaskSelection {
 
 
 impl TaskSelection {
-    fn should_continue(&self) -> bool {
+    fn should_proceed(&self) -> bool {
         match self {
             TaskSelection::OnlyFromRunnable => true,
             TaskSelection::OnlyFromContended(failure_count) => *failure_count < 2,
@@ -1540,9 +1540,9 @@ impl ScheduleStage {
 
             loop {
                 let executing_like_count = executing_queue_count + provisioning_tracker_count;
-                if executing_like_count < max_executing_queue_count {
+                let mut selection = TaskSelection::OnlyFromContended(0);
+                if selection.should_proceed() && executing_like_count < max_executing_queue_count {
                     let prefer_immediate = true; //provisioning_tracker_count / 4 > executing_queue_count;
-                    let mut selection = TaskSelection::OnlyFromContended(0);
 
                     if let Some(ee) = Self::schedule_next_execution(
                         ast,
@@ -1573,9 +1573,9 @@ impl ScheduleStage {
                         }
                     }
                 }
-                while executing_queue_count + provisioning_tracker_count < max_executing_queue_count {
+                let mut selection = TaskSelection::OnlyFromRunnable;
+                while selection.should_proceed() && executing_queue_count + provisioning_tracker_count < max_executing_queue_count {
                     let prefer_immediate = true; //provisioning_tracker_count / 4 > executing_queue_count;
-                    let mut selection = TaskSelection::OnlyFromRunnable;
 
                     let maybe_ee = Self::schedule_next_execution(
                         ast,

From f6ccca2d2314983bd73597333fc46ef97c1d42b2 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 11 Sep 2022 21:52:52 +0900
Subject: [PATCH 2434/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 5d22719a218d66..581befa659e51e 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1540,7 +1540,7 @@ impl ScheduleStage {
 
             loop {
                 let executing_like_count = executing_queue_count + provisioning_tracker_count;
-                let mut selection = TaskSelection::OnlyFromContended(0);
+                let mut selection = TaskSelection::OnlyFromContended(Default::default());
                 if selection.should_proceed() && executing_like_count < max_executing_queue_count {
                     let prefer_immediate = true; //provisioning_tracker_count / 4 > executing_queue_count;
 

From 187d5c574dea2cadcbb3118c0aab9215d2941d83 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 11 Sep 2022 21:56:49 +0900
Subject: [PATCH 2435/3199] save

---
 scheduler/src/lib.rs | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 581befa659e51e..9f5b94a958b59e 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1042,6 +1042,10 @@ impl ScheduleStage {
 
                 if unlockable_count > 0 {
                     *failed_lock_count += 1;
+                    if let TaskSelection::OnlyFromContended(ref mut failure_count) = task_selection {
+                        failure_count += 1;
+                    }
+
                     //trace!("reset_lock_for_failed_execution(): {:?} {}", (&unique_weight, from_runnable), next_task.tx.0.signature());
                     Self::reset_lock_for_failed_execution(
                         ast,

From c91f1b5825b410a2ad20c3d84d3665fa9fb61095 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 11 Sep 2022 21:57:02 +0900
Subject: [PATCH 2436/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 9f5b94a958b59e..aca892dcc31618 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1043,7 +1043,7 @@ impl ScheduleStage {
                 if unlockable_count > 0 {
                     *failed_lock_count += 1;
                     if let TaskSelection::OnlyFromContended(ref mut failure_count) = task_selection {
-                        failure_count += 1;
+                        *failure_count += 1;
                     }
 
                     //trace!("reset_lock_for_failed_execution(): {:?} {}", (&unique_weight, from_runnable), next_task.tx.0.signature());

From e0e698159ab23eb3d897969f460091a9d3b52a05 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 11 Sep 2022 21:57:19 +0900
Subject: [PATCH 2437/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index aca892dcc31618..c421b67db203f1 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1042,7 +1042,7 @@ impl ScheduleStage {
 
                 if unlockable_count > 0 {
                     *failed_lock_count += 1;
-                    if let TaskSelection::OnlyFromContended(ref mut failure_count) = task_selection {
+                    if let TaskSelection::OnlyFromContended(ref failure_count) = task_selection {
                         *failure_count += 1;
                     }
 

From aac4b8885a0c7dcfb3e60e641bccf26c07c7c5ce Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 11 Sep 2022 21:57:29 +0900
Subject: [PATCH 2438/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index c421b67db203f1..d250b0e0a90982 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1042,7 +1042,7 @@ impl ScheduleStage {
 
                 if unlockable_count > 0 {
                     *failed_lock_count += 1;
-                    if let TaskSelection::OnlyFromContended(ref failure_count) = task_selection {
+                    if let TaskSelection::OnlyFromContended(failure_count) = task_selection {
                         *failure_count += 1;
                     }
 

From aa37353d3afba838a8621dc109caeff09493b29a Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 11 Sep 2022 21:58:05 +0900
Subject: [PATCH 2439/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index d250b0e0a90982..aca892dcc31618 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1042,7 +1042,7 @@ impl ScheduleStage {
 
                 if unlockable_count > 0 {
                     *failed_lock_count += 1;
-                    if let TaskSelection::OnlyFromContended(failure_count) = task_selection {
+                    if let TaskSelection::OnlyFromContended(ref mut failure_count) = task_selection {
                         *failure_count += 1;
                     }
 

From 3816eccf7edd30b63273867992c5324dc25be20e Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 11 Sep 2022 22:00:56 +0900
Subject: [PATCH 2440/3199] save

---
 scheduler/src/lib.rs | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index aca892dcc31618..4d7d0807e9b822 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1123,7 +1123,8 @@ impl ScheduleStage {
                             .insert(next_task.stuck_task_id(), removed);
                         assert!(a.is_none());
                         */
-                        address_book.uncontended_task_ids.insert(next_task.unique_weight, next_task);
+
+                        //address_book.uncontended_task_ids.insert(next_task.unique_weight, next_task);
                         
                         break;
                     } else {
@@ -1620,6 +1621,8 @@ impl ScheduleStage {
                     }
                 }
 
+                // this is hand-written unrolled select loop to reduce sched_yield (futex) as much
+                // as possible, taking advangage of only 2 select target for our impl
                 if first_iteration {
                     first_iteration = false;
                     (from_len, from_exec_len) = (from_prev.len(), from_exec.len());

From 2ff36b6c5c207539b258885b36632b569d99ac7f Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 11 Sep 2022 22:04:57 +0900
Subject: [PATCH 2441/3199] save

---
 scheduler/src/lib.rs | 20 ++++++++++++++------
 1 file changed, 14 insertions(+), 6 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 4d7d0807e9b822..46b20a75eab8b6 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1544,12 +1544,11 @@ impl ScheduleStage {
             let (mut from_len, mut from_exec_len) = (0, 0);
 
             loop {
-                let executing_like_count = executing_queue_count + provisioning_tracker_count;
                 let mut selection = TaskSelection::OnlyFromContended(Default::default());
-                if selection.should_proceed() && executing_like_count < max_executing_queue_count {
+                while selection.should_proceed() && executing_queue_count + provisioning_tracker_count < max_executing_queue_count {
                     let prefer_immediate = true; //provisioning_tracker_count / 4 > executing_queue_count;
 
-                    if let Some(ee) = Self::schedule_next_execution(
+                    let maybe_ee = Self::schedule_next_execution(
                         ast,
                         &task_sender,
                         runnable_queue,
@@ -1562,11 +1561,20 @@ impl ScheduleStage {
                         &mut provisioning_tracker_count,
                         &mut selection,
                         &mut failed_lock_count,
-                    ) {
+                    );
+                    if let Some(ee) = maybe_ee {
                         executing_queue_count = executing_queue_count.checked_add(1).unwrap();
                         to_execute_substage.send(ExecutablePayload(ee)).unwrap();
+                        debug!("schedule_once id_{:016x} [C] ch(prev: {}, exec: {}|{}), r: {}, u/c: {}/{}, (imm+provi)/max: ({}+{})/{} s: {} l(s+f): {}+{}", random_id, (if from_disconnected { "-".to_string() } else { format!("{}", from_prev.len()) }), to_execute_substage.len(), from_exec.len(), runnable_queue.task_count(), address_book.uncontended_task_ids.len(), contended_count, executing_queue_count, provisioning_tracker_count, max_executing_queue_count, address_book.stuck_tasks.len(), processed_count, failed_lock_count);
+                    } else {
+                        debug!("schedule_once id_{:016x} [C] ch(prev: {}, exec: {}|{}), r: {}, u/c: {}/{}, (imm+provi)/max: ({}+{})/{} s: {} l(s+f): {}+{}", random_id, (if from_disconnected { "-".to_string() } else { format!("{}", from_prev.len()) }), to_execute_substage.len(), from_exec.len(), runnable_queue.task_count(), address_book.uncontended_task_ids.len(), contended_count, executing_queue_count, provisioning_tracker_count, max_executing_queue_count, address_book.stuck_tasks.len(), processed_count, failed_lock_count);
+                        break;
                     }
-                    debug!("schedule_once id_{:016x} [C] ch(prev: {}, exec: {}|{}), r: {}, u/c: {}/{}, (imm+provi)/max: ({}+{})/{} s: {} l(s+f): {}+{}", random_id, (if from_disconnected { "-".to_string() } else { format!("{}", from_prev.len()) }), to_execute_substage.len(), from_exec.len(), runnable_queue.task_count(), address_book.uncontended_task_ids.len(), contended_count, executing_queue_count, provisioning_tracker_count, max_executing_queue_count, address_book.stuck_tasks.len(), processed_count, failed_lock_count);
+                    if !from_exec.is_empty() {
+                        trace!("abort aggressive contended queue processing due to non-empty from_exec");
+                        break;
+                    }
+
                     interval_count += 1;
                     if interval_count % 100 == 0 {
                         let elapsed = last_time.elapsed();
@@ -1605,7 +1613,7 @@ impl ScheduleStage {
                         break;
                     }
                     if !from_exec.is_empty() {
-                        trace!("abort aggressive readable queue processing due to non-empty from_exec");
+                        trace!("abort aggressive runnable queue processing due to non-empty from_exec");
                         break;
                     }
 

From 0a6100756cb7ff1aec939af37aa642c426566330 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 12 Sep 2022 09:24:45 +0900
Subject: [PATCH 2442/3199] save

---
 runtime/src/bank.rs  | 16 ++++++++++++----
 scheduler/src/lib.rs |  2 +-
 2 files changed, 13 insertions(+), 5 deletions(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 352c52d613c767..4cf1ab28a91ebf 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1236,6 +1236,7 @@ impl Scheduler {
 
 impl Default for Scheduler {
     fn default() -> Self {
+        let start = Instant::now();
         let mut address_book = solana_scheduler::AddressBook::default();
         let preloader = Arc::new(address_book.preloader());
         let (transaction_sender, transaction_receiver) = crossbeam_channel::unbounded();
@@ -1305,7 +1306,9 @@ impl Default for Scheduler {
                     ee.cu = details.executed_units;
                     send_metrics.then(|| format!("{:?}", details.status))
                 } else {
-                    error!("found odd tx error: {:?}", tx_result);
+                    let sig = ee.task.tx.0.signature().to_string();
+                    error!("found odd tx error: slot: {}, signature: {}, {:?}", slot, sig, tx_result);
+
                     send_metrics.then(|| format!("{:?}", tx_result))
                 };
 
@@ -1376,7 +1379,7 @@ impl Default for Scheduler {
             Ok(())
         }).unwrap();
 
-        Self {
+        let s = Self {
             scheduler_thread_handle: Some(scheduler_thread_handle),
             executing_thread_handles: Some(executing_thread_handles),
             error_collector_thread_handle: Some(error_collector_thread_handle),
@@ -1387,6 +1390,9 @@ impl Default for Scheduler {
             bank,
             slot: Default::default(),
         }
+        info!("scheduler: setup done with {}us", start.elapsed());
+
+        s
     }
 }
 
@@ -4088,13 +4094,15 @@ impl Bank {
     pub fn register_recent_blockhash(&self, blockhash: &Hash) {
         info!("register_recent_blockhash: slot: {} reinitializing the scheduler: start", self.slot());
 
-        self.wait_for_scheduler().unwrap();
+        let last_error = self.wait_for_scheduler();
 
         // Only acquire the write lock for the blockhash queue on block boundaries because
         // readers can starve this write lock acquisition and ticks would be slowed down too
         // much if the write lock is acquired for each tick.
         let mut w_blockhash_queue = self.blockhash_queue.write().unwrap();
-        *self.scheduler.write().unwrap() = Default::default();
+        let mut new_scheduler = Default::default();
+        new_scheduler.errors.lock().unwrap().push(last_error);
+        *self.scheduler.write().unwrap() = new_scheduler;
 
         info!("register_recent_blockhash: slot: {} reinitializing the scheduler: end", self.slot());
 
diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 46b20a75eab8b6..4edacef5edf684 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1686,7 +1686,7 @@ impl ScheduleStage {
 
         let elapsed = start_time.elapsed();
         let elapsed2 = elapsed.as_micros();
-        info!("schedule_once:final id_{:016x} (from_disconnected: {}, from_exec_disconnected: {}, no_more_work: {}) ch(prev: {}, exec: {}|{}), runnnable: {}, contended: {}, (immediate+provisional)/max: ({}+{})/{} uncontended: {} stuck: {} overall: {}txs/{}us={}tps!", random_id, from_disconnected, from_exec_disconnected, no_more_work, from_prev.len(), to_execute_substage.len(), from_exec.len(), runnable_queue.task_count(), contended_count, executing_queue_count, provisioning_tracker_count, max_executing_queue_count, address_book.uncontended_task_ids.len(), address_book.stuck_tasks.len(), processed_count, elapsed.as_micros(), 1_000_000_u128*(processed_count as u128)/elapsed2);
+        info!("schedule_once:final id_{:016x} (from_disconnected: {}, from_exec_disconnected: {}, no_more_work: {}) ch(prev: {}, exec: {}|{}), runnnable: {}, contended: {}, (immediate+provisional)/max: ({}+{})/{} uncontended: {} stuck: {} miss: {}, overall: {}txs/{}us={}tps!", random_id, from_disconnected, from_exec_disconnected, no_more_work, from_prev.len(), to_execute_substage.len(), from_exec.len(), runnable_queue.task_count(), contended_count, executing_queue_count, provisioning_tracker_count, max_executing_queue_count, address_book.uncontended_task_ids.len(), address_book.stuck_tasks.len(), failed_lock_count, processed_count, elapsed.as_micros(), 1_000_000_u128*(processed_count as u128)/elapsed2);
     }
 
     pub fn run(

From af5ed69dc2c007071da3686da9489a03738eabcd Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 12 Sep 2022 09:25:01 +0900
Subject: [PATCH 2443/3199] save

---
 runtime/src/bank.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 4cf1ab28a91ebf..d23d4edf8a8806 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1389,7 +1389,7 @@ impl Default for Scheduler {
             errors,
             bank,
             slot: Default::default(),
-        }
+        };
         info!("scheduler: setup done with {}us", start.elapsed());
 
         s

From 6e0522670d052718b8bdecae612c2a1262890244 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 12 Sep 2022 09:25:18 +0900
Subject: [PATCH 2444/3199] save

---
 runtime/src/bank.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index d23d4edf8a8806..d7831f6cf0491d 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -4100,7 +4100,7 @@ impl Bank {
         // readers can starve this write lock acquisition and ticks would be slowed down too
         // much if the write lock is acquired for each tick.
         let mut w_blockhash_queue = self.blockhash_queue.write().unwrap();
-        let mut new_scheduler = Default::default();
+        let mut new_scheduler = Scheduler::default();
         new_scheduler.errors.lock().unwrap().push(last_error);
         *self.scheduler.write().unwrap() = new_scheduler;
 

From caac245c8776be33466f68b3151cfb39c4bcbe0c Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 12 Sep 2022 09:25:33 +0900
Subject: [PATCH 2445/3199] save

---
 runtime/src/bank.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index d7831f6cf0491d..1f5fbf3972b1a9 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1390,7 +1390,7 @@ impl Default for Scheduler {
             bank,
             slot: Default::default(),
         };
-        info!("scheduler: setup done with {}us", start.elapsed());
+        info!("scheduler: setup done with {}us", start.elapsed().as_micros());
 
         s
     }

From 24c58d87a256aebfec2e5f4d088b6659188b4e4c Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 12 Sep 2022 09:28:19 +0900
Subject: [PATCH 2446/3199] save

---
 runtime/src/bank.rs | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 1f5fbf3972b1a9..0e091fcfc6e69c 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -4094,14 +4094,16 @@ impl Bank {
     pub fn register_recent_blockhash(&self, blockhash: &Hash) {
         info!("register_recent_blockhash: slot: {} reinitializing the scheduler: start", self.slot());
 
-        let last_error = self.wait_for_scheduler();
+        let maybe_last_error = self.wait_for_scheduler();
 
         // Only acquire the write lock for the blockhash queue on block boundaries because
         // readers can starve this write lock acquisition and ticks would be slowed down too
         // much if the write lock is acquired for each tick.
         let mut w_blockhash_queue = self.blockhash_queue.write().unwrap();
         let mut new_scheduler = Scheduler::default();
-        new_scheduler.errors.lock().unwrap().push(last_error);
+        if maybe_last_error.is_error() {
+            new_scheduler.errors.lock().unwrap().push(maybe_last_error);
+        }
         *self.scheduler.write().unwrap() = new_scheduler;
 
         info!("register_recent_blockhash: slot: {} reinitializing the scheduler: end", self.slot());

From e5a01bd2d6aac8198c541be076abbb074a800f8c Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 12 Sep 2022 09:28:53 +0900
Subject: [PATCH 2447/3199] save

---
 runtime/src/bank.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 0e091fcfc6e69c..3986d9a5d753ac 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -4101,7 +4101,7 @@ impl Bank {
         // much if the write lock is acquired for each tick.
         let mut w_blockhash_queue = self.blockhash_queue.write().unwrap();
         let mut new_scheduler = Scheduler::default();
-        if maybe_last_error.is_error() {
+        if maybe_last_error.is_err() {
             new_scheduler.errors.lock().unwrap().push(maybe_last_error);
         }
         *self.scheduler.write().unwrap() = new_scheduler;

From 57ab7583a3f4d3aee5f917a84775a81341177a46 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 12 Sep 2022 09:38:25 +0900
Subject: [PATCH 2448/3199] save

---
 scheduler/src/lib.rs | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 4edacef5edf684..896ed3477ebc88 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -884,7 +884,7 @@ impl TaskSelection {
     fn should_proceed(&self) -> bool {
         match self {
             TaskSelection::OnlyFromRunnable => true,
-            TaskSelection::OnlyFromContended(failure_count) => *failure_count < 2,
+            TaskSelection::OnlyFromContended(retry_count) => retry_count > 0,
         }
     }
 
@@ -1042,8 +1042,8 @@ impl ScheduleStage {
 
                 if unlockable_count > 0 {
                     *failed_lock_count += 1;
-                    if let TaskSelection::OnlyFromContended(ref mut failure_count) = task_selection {
-                        *failure_count += 1;
+                    if let TaskSelection::OnlyFromContended(ref mut retry_count) = task_selection {
+                        *retry_count -= 1;
                     }
 
                     //trace!("reset_lock_for_failed_execution(): {:?} {}", (&unique_weight, from_runnable), next_task.tx.0.signature());
@@ -1544,7 +1544,7 @@ impl ScheduleStage {
             let (mut from_len, mut from_exec_len) = (0, 0);
 
             loop {
-                let mut selection = TaskSelection::OnlyFromContended(Default::default());
+                let mut selection = TaskSelection::OnlyFromContended(if from_disconnected { usize::max_value() } else { 2 });
                 while selection.should_proceed() && executing_queue_count + provisioning_tracker_count < max_executing_queue_count {
                     let prefer_immediate = true; //provisioning_tracker_count / 4 > executing_queue_count;
 

From 7d5ae1a252a42b25c5c2d65fd18204a5d522e005 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 12 Sep 2022 09:39:08 +0900
Subject: [PATCH 2449/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 896ed3477ebc88..70b4ffebdbc4d3 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -884,7 +884,7 @@ impl TaskSelection {
     fn should_proceed(&self) -> bool {
         match self {
             TaskSelection::OnlyFromRunnable => true,
-            TaskSelection::OnlyFromContended(retry_count) => retry_count > 0,
+            TaskSelection::OnlyFromContended(retry_count) => *retry_count > 0,
         }
     }
 

From 98e9311a805fbeef422fe085fd7e753f1288d535 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 12 Sep 2022 09:46:58 +0900
Subject: [PATCH 2450/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 70b4ffebdbc4d3..4e94fd25f0bbee 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1544,7 +1544,7 @@ impl ScheduleStage {
             let (mut from_len, mut from_exec_len) = (0, 0);
 
             loop {
-                let mut selection = TaskSelection::OnlyFromContended(if from_disconnected { usize::max_value() } else { 2 });
+                let mut selection = TaskSelection::OnlyFromContended(if from_disconnected && runnable_queue.is_empty() { usize::max_value() } else { 2 });
                 while selection.should_proceed() && executing_queue_count + provisioning_tracker_count < max_executing_queue_count {
                     let prefer_immediate = true; //provisioning_tracker_count / 4 > executing_queue_count;
 

From 79a4499246aa3d671f65709b711218708676ad85 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 12 Sep 2022 09:47:20 +0900
Subject: [PATCH 2451/3199] save

---
 scheduler/src/lib.rs | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 4e94fd25f0bbee..cb8f0582aed985 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -807,6 +807,10 @@ impl TaskQueue {
     fn task_count(&self) -> usize {
         self.tasks.len()
     }
+
+    fn is_empty(&self) -> bool {
+        self.tasks.is_empty()
+    }
 }
 
 #[inline(never)]

From e6412790a14c8131e4193d270ac24a236a2c1ab7 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 12 Sep 2022 09:47:50 +0900
Subject: [PATCH 2452/3199] save

---
 scheduler/src/lib.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index cb8f0582aed985..263f291e4bb829 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -808,7 +808,7 @@ impl TaskQueue {
         self.tasks.len()
     }
 
-    fn is_empty(&self) -> bool {
+    fn has_no_task(&self) -> bool {
         self.tasks.is_empty()
     }
 }
@@ -1548,7 +1548,7 @@ impl ScheduleStage {
             let (mut from_len, mut from_exec_len) = (0, 0);
 
             loop {
-                let mut selection = TaskSelection::OnlyFromContended(if from_disconnected && runnable_queue.is_empty() { usize::max_value() } else { 2 });
+                let mut selection = TaskSelection::OnlyFromContended(if from_disconnected && runnable_queue.has_no_task() { usize::max_value() } else { 2 });
                 while selection.should_proceed() && executing_queue_count + provisioning_tracker_count < max_executing_queue_count {
                     let prefer_immediate = true; //provisioning_tracker_count / 4 > executing_queue_count;
 

From bffb8eee36782e59b530e0cdb53d73a4360266b3 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 12 Sep 2022 09:58:08 +0900
Subject: [PATCH 2453/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 263f291e4bb829..65dfb06d2c4c93 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1047,7 +1047,7 @@ impl ScheduleStage {
                 if unlockable_count > 0 {
                     *failed_lock_count += 1;
                     if let TaskSelection::OnlyFromContended(ref mut retry_count) = task_selection {
-                        *retry_count -= 1;
+                        *retry_count = retry_count.checked_sub(1).unwrap();
                     }
 
                     //trace!("reset_lock_for_failed_execution(): {:?} {}", (&unique_weight, from_runnable), next_task.tx.0.signature());

From edc96a25c102b3c42054bb14f5a0f90d6f34dcb7 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 12 Sep 2022 12:56:41 +0900
Subject: [PATCH 2454/3199] save

---
 runtime/Cargo.toml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/runtime/Cargo.toml b/runtime/Cargo.toml
index 8b2ea6ce652572..63c0c4807ba66c 100644
--- a/runtime/Cargo.toml
+++ b/runtime/Cargo.toml
@@ -17,6 +17,7 @@ bv = { version = "0.11.1", features = ["serde"] }
 bytemuck = "1.11.0"
 byteorder = "1.4.3"
 bzip2 = "0.4.3"
+cpu-time = "1.0.0"
 crossbeam-channel = "0.5"
 dashmap = { version = "4.0.2", features = ["rayon", "raw-api"] }
 dir-diff = "0.3.2"

From b47b32834e1614076bbf4854473f5ef0a6852a7a Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 12 Sep 2022 13:10:49 +0900
Subject: [PATCH 2455/3199] save

---
 Cargo.lock         | 11 +++++++++++
 runtime/Cargo.toml |  1 +
 2 files changed, 12 insertions(+)

diff --git a/Cargo.lock b/Cargo.lock
index 5c03faca5f4bc1..05d3abee34217d 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -959,6 +959,16 @@ dependencies = [
  "winapi 0.2.8",
 ]
 
+[[package]]
+name = "cpu-time"
+version = "1.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e9e393a7668fe1fad3075085b86c781883000b4ede868f43627b34a87c8b7ded"
+dependencies = [
+ "libc",
+ "winapi 0.3.9",
+]
+
 [[package]]
 name = "cpufeatures"
 version = "0.2.1"
@@ -6135,6 +6145,7 @@ dependencies = [
  "bytemuck",
  "byteorder",
  "bzip2",
+ "cpu-time",
  "crossbeam-channel",
  "dashmap 4.0.2",
  "dir-diff",
diff --git a/runtime/Cargo.toml b/runtime/Cargo.toml
index 63c0c4807ba66c..1bd2024e0d292a 100644
--- a/runtime/Cargo.toml
+++ b/runtime/Cargo.toml
@@ -63,6 +63,7 @@ symlink = "0.1.0"
 tar = "0.4.38"
 tempfile = "3.3.0"
 thiserror = "1.0"
+thread-priority = "0.9.2"
 zstd = "0.11.2"
 
 [lib]

From c99c88de6145a34a6c2a2de9d75d450432344a9e Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 12 Sep 2022 13:11:36 +0900
Subject: [PATCH 2456/3199] save

---
 Cargo.lock          | 13 +++++++++++++
 runtime/src/bank.rs |  2 +-
 2 files changed, 14 insertions(+), 1 deletion(-)

diff --git a/Cargo.lock b/Cargo.lock
index 05d3abee34217d..4d59629f989a50 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -6196,6 +6196,7 @@ dependencies = [
  "tar",
  "tempfile",
  "thiserror",
+ "thread-priority",
  "zstd",
 ]
 
@@ -7263,6 +7264,18 @@ dependencies = [
  "syn 1.0.98",
 ]
 
+[[package]]
+name = "thread-priority"
+version = "0.9.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5a8a950b52fd40d98ac6ed41c7fa9e8dd62b131f48b74f418e810476b01a7460"
+dependencies = [
+ "cfg-if 1.0.0",
+ "libc",
+ "log",
+ "winapi 0.3.9",
+]
+
 [[package]]
 name = "thread-scoped"
 version = "1.0.2"
diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 3986d9a5d753ac..1b0ec75383f6ac 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -4100,7 +4100,7 @@ impl Bank {
         // readers can starve this write lock acquisition and ticks would be slowed down too
         // much if the write lock is acquired for each tick.
         let mut w_blockhash_queue = self.blockhash_queue.write().unwrap();
-        let mut new_scheduler = Scheduler::default();
+        let new_scheduler = Scheduler::default();
         if maybe_last_error.is_err() {
             new_scheduler.errors.lock().unwrap().push(maybe_last_error);
         }

From e9527131399f14650ac23011c13363385c691452 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 12 Sep 2022 13:12:51 +0900
Subject: [PATCH 2457/3199] save

---
 runtime/src/bank.rs | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 1b0ec75383f6ac..7abd4f1cd36d5a 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1357,6 +1357,7 @@ impl Default for Scheduler {
         }).unwrap();
 
         let scheduler_thread_handle = std::thread::Builder::new().name("solScheduler".to_string()).spawn(move || {
+            let started =  cpu_time::ThreadTime::now();
             let mut runnable_queue = solana_scheduler::TaskQueue::default();
             let max_executing_queue_count = std::env::var("MAX_EXECUTING_QUEUE_COUNT")
                 .unwrap_or(format!("{}", 1))
@@ -1376,7 +1377,7 @@ impl Default for Scheduler {
             drop(scheduled_ee_sender);
             drop(processed_ee_receiver);
 
-            Ok(())
+            Ok(started)
         }).unwrap();
 
         let s = Self {

From b7619f611a909a6c4e3c43d5685b436a5db05c6f Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 12 Sep 2022 13:13:15 +0900
Subject: [PATCH 2458/3199] save

---
 runtime/src/bank.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 7abd4f1cd36d5a..c3bdc8ab62946c 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1377,7 +1377,7 @@ impl Default for Scheduler {
             drop(scheduled_ee_sender);
             drop(processed_ee_receiver);
 
-            Ok(started)
+            Ok(started.elapsed())
         }).unwrap();
 
         let s = Self {

From 1f7c3c6451296b7b327f8b8d7a979a28726f4372 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 12 Sep 2022 13:13:41 +0900
Subject: [PATCH 2459/3199] save

---
 runtime/src/bank.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index c3bdc8ab62946c..08faaeb2b38d70 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1198,7 +1198,7 @@ impl AbiExample for BuiltinPrograms {
 
 #[derive(Debug)]
 struct Scheduler {
-    scheduler_thread_handle: Option<std::thread::JoinHandle<Result<()>>>,
+    scheduler_thread_handle: Option<std::thread::JoinHandle<Result<Duration>>>,
     executing_thread_handles: Option<Vec<std::thread::JoinHandle<Result<()>>>>,
     error_collector_thread_handle: Option<std::thread::JoinHandle<Result<()>>>,
     transaction_sender: Option<crossbeam_channel::Sender<solana_scheduler::SchedulablePayload>>,

From c0969a5360a5b53e04b4c4b9cae326df5ed480a5 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 12 Sep 2022 13:14:15 +0900
Subject: [PATCH 2460/3199] save

---
 runtime/src/bank.rs | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 08faaeb2b38d70..56ef0a98451de7 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1342,6 +1342,8 @@ impl Default for Scheduler {
         let errors_in_collector_thread = Arc::clone(&errors);
 
         let error_collector_thread_handle = std::thread::Builder::new().name(format!("solScErrCol{:02}", 0)).spawn(move || {
+            let started =  cpu_time::ThreadTime::now();
+
             while let Ok(solana_scheduler::ExaminablePayload(mut ee)) = retired_ee_receiver.recv() {
                 if ee.is_aborted() {
                     warn!(
@@ -1353,7 +1355,7 @@ impl Default for Scheduler {
 
                 drop(ee);
             }
-            Ok(())
+            Ok(started.elapsed())
         }).unwrap();
 
         let scheduler_thread_handle = std::thread::Builder::new().name("solScheduler".to_string()).spawn(move || {

From 1e896f2136b9f2f7c217d24fba1dd402797b5973 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 12 Sep 2022 13:14:30 +0900
Subject: [PATCH 2461/3199] save

---
 runtime/src/bank.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 56ef0a98451de7..860a6d5c9a0c6a 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1200,7 +1200,7 @@ impl AbiExample for BuiltinPrograms {
 struct Scheduler {
     scheduler_thread_handle: Option<std::thread::JoinHandle<Result<Duration>>>,
     executing_thread_handles: Option<Vec<std::thread::JoinHandle<Result<()>>>>,
-    error_collector_thread_handle: Option<std::thread::JoinHandle<Result<()>>>,
+    error_collector_thread_handle: Option<std::thread::JoinHandle<Result<Duration>>>,
     transaction_sender: Option<crossbeam_channel::Sender<solana_scheduler::SchedulablePayload>>,
     preloader: Arc<solana_scheduler::Preloader>,
     graceful_stop_initiated: bool,

From 5d5fa630eb8ebead86b9844dccf02a0147f70d08 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 12 Sep 2022 13:15:12 +0900
Subject: [PATCH 2462/3199] save

---
 runtime/src/bank.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 860a6d5c9a0c6a..622fd91fcdcdd8 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1199,7 +1199,7 @@ impl AbiExample for BuiltinPrograms {
 #[derive(Debug)]
 struct Scheduler {
     scheduler_thread_handle: Option<std::thread::JoinHandle<Result<Duration>>>,
-    executing_thread_handles: Option<Vec<std::thread::JoinHandle<Result<()>>>>,
+    executing_thread_handles: Option<Vec<std::thread::JoinHandle<Result<Duration>>>>,
     error_collector_thread_handle: Option<std::thread::JoinHandle<Result<Duration>>>,
     transaction_sender: Option<crossbeam_channel::Sender<solana_scheduler::SchedulablePayload>>,
     preloader: Arc<solana_scheduler::Preloader>,

From 642ccc357b516210d4950237619a697b250fe5ea Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 12 Sep 2022 13:15:44 +0900
Subject: [PATCH 2463/3199] save

---
 runtime/src/bank.rs | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 622fd91fcdcdd8..f16fc9222bafe5 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1258,6 +1258,7 @@ impl Default for Scheduler {
             let bank = bank.clone();
 
             std::thread::Builder::new().name(format!("solScExLane{:02}", thx)).spawn(move || {
+            let started =  cpu_time::ThreadTime::now();
             let current_thread_name = std::thread::current().name().unwrap().to_string();
 
             while let Ok(solana_scheduler::ExecutablePayload(mut ee)) = scheduled_ee_receiver.recv() {
@@ -1335,7 +1336,7 @@ impl Default for Scheduler {
                 //ee.reindex_with_address_book();
                 processed_ee_sender.send(solana_scheduler::UnlockablePayload(ee)).unwrap();
             }
-            Ok(())
+            Ok(started.elapsed())
         }).unwrap()}).collect();
 
         let errors = Arc::new(std::sync::Mutex::new(Vec::new()));

From cbb111b0fcf54c09da728b94eaa078b5e75c5236 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 12 Sep 2022 13:21:07 +0900
Subject: [PATCH 2464/3199] save

---
 runtime/src/bank.rs | 12 +++++++-----
 1 file changed, 7 insertions(+), 5 deletions(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index f16fc9222bafe5..33e47ff608f93e 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1411,13 +1411,15 @@ impl Scheduler {
         info!("Scheduler::gracefully_stop(): waiting..");
         let transaction_sender = self.transaction_sender.take().unwrap();
         drop(transaction_sender);
-        for executing_thread_handle in self.executing_thread_handles.take().unwrap() {
-            executing_thread_handle.join().unwrap()?;
-        }
+        let executing_thread_cpu_us = self.executing_thread_handles.take().unwrap().map(|executing_thread_handle| {
+            executing_thread_handle.join().unwrap()?.as_micros();
+        });
         let h = self.scheduler_thread_handle.take().unwrap();
-        h.join().unwrap()?;
+        let scheduler_thread_cpu_us = h.join().unwrap()?.as_micros();
         let h = self.error_collector_thread_handle.take().unwrap();
-        h.join().unwrap()?;
+        let error_collector_thread_cpu_us = h.join().unwrap()?.as_micros();
+
+        info!("Scheduler::gracefully_stop(): stopped: schduler: {}, error_collector: {}, executing: {} = {:?}", scheduler_thread_cpu_us, error_collector_thread_cpu_us, executing_thread_cpu_us.iter().sum(), &executing_thread_cpu_us);
 
         Ok(())
     }

From 4b68e35f6a5fb9625a957513ad604c647ae1bd52 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 12 Sep 2022 13:21:44 +0900
Subject: [PATCH 2465/3199] save

---
 runtime/src/bank.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 33e47ff608f93e..ac3e5bce6b3ec8 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1411,7 +1411,7 @@ impl Scheduler {
         info!("Scheduler::gracefully_stop(): waiting..");
         let transaction_sender = self.transaction_sender.take().unwrap();
         drop(transaction_sender);
-        let executing_thread_cpu_us = self.executing_thread_handles.take().unwrap().map(|executing_thread_handle| {
+        let executing_thread_cpu_us = self.executing_thread_handles.take().unwrap().iter().map(|executing_thread_handle| {
             executing_thread_handle.join().unwrap()?.as_micros();
         });
         let h = self.scheduler_thread_handle.take().unwrap();

From 4a1aebf755502123eb11db30b11b969999bcc0d2 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 12 Sep 2022 13:22:00 +0900
Subject: [PATCH 2466/3199] save

---
 runtime/src/bank.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index ac3e5bce6b3ec8..076fa0ee635e6d 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1412,7 +1412,7 @@ impl Scheduler {
         let transaction_sender = self.transaction_sender.take().unwrap();
         drop(transaction_sender);
         let executing_thread_cpu_us = self.executing_thread_handles.take().unwrap().iter().map(|executing_thread_handle| {
-            executing_thread_handle.join().unwrap()?.as_micros();
+            executing_thread_handle.join().unwrap()?.as_micros()
         });
         let h = self.scheduler_thread_handle.take().unwrap();
         let scheduler_thread_cpu_us = h.join().unwrap()?.as_micros();

From 108ebee677276ef9feab39f9cae2af80ed4b69ff Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 12 Sep 2022 13:23:13 +0900
Subject: [PATCH 2467/3199] save

---
 runtime/src/bank.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 076fa0ee635e6d..831865098587cb 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1412,7 +1412,7 @@ impl Scheduler {
         let transaction_sender = self.transaction_sender.take().unwrap();
         drop(transaction_sender);
         let executing_thread_cpu_us = self.executing_thread_handles.take().unwrap().iter().map(|executing_thread_handle| {
-            executing_thread_handle.join().unwrap()?.as_micros()
+            executing_thread_handle.join().unwrap().map(|u| u.as_micros())
         });
         let h = self.scheduler_thread_handle.take().unwrap();
         let scheduler_thread_cpu_us = h.join().unwrap()?.as_micros();

From e48fc981f0f5f13673f92e5405210ccc32c64910 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 12 Sep 2022 13:25:57 +0900
Subject: [PATCH 2468/3199] save

---
 runtime/src/bank.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 831865098587cb..2873c4030e3048 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1411,7 +1411,7 @@ impl Scheduler {
         info!("Scheduler::gracefully_stop(): waiting..");
         let transaction_sender = self.transaction_sender.take().unwrap();
         drop(transaction_sender);
-        let executing_thread_cpu_us = self.executing_thread_handles.take().unwrap().iter().map(|executing_thread_handle| {
+        let executing_thread_cpu_us: usize = self.executing_thread_handles.take().unwrap().iter().collect(|executing_thread_handle| {
             executing_thread_handle.join().unwrap().map(|u| u.as_micros())
         });
         let h = self.scheduler_thread_handle.take().unwrap();

From 5ab174325083c7a9a1766c43300e14a764896ac0 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 12 Sep 2022 13:26:23 +0900
Subject: [PATCH 2469/3199] save

---
 runtime/src/bank.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 2873c4030e3048..e298d05816d8a1 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1411,9 +1411,9 @@ impl Scheduler {
         info!("Scheduler::gracefully_stop(): waiting..");
         let transaction_sender = self.transaction_sender.take().unwrap();
         drop(transaction_sender);
-        let executing_thread_cpu_us: usize = self.executing_thread_handles.take().unwrap().iter().collect(|executing_thread_handle| {
+        let executing_thread_cpu_us: usize = self.executing_thread_handles.take().unwrap().iter().map(|executing_thread_handle| {
             executing_thread_handle.join().unwrap().map(|u| u.as_micros())
-        });
+        }).collect();
         let h = self.scheduler_thread_handle.take().unwrap();
         let scheduler_thread_cpu_us = h.join().unwrap()?.as_micros();
         let h = self.error_collector_thread_handle.take().unwrap();

From 22513d7e411b05fd3c67d815371417810f4247cc Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 12 Sep 2022 13:26:46 +0900
Subject: [PATCH 2470/3199] save

---
 runtime/src/bank.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index e298d05816d8a1..b6b18587ff9b1a 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1411,7 +1411,7 @@ impl Scheduler {
         info!("Scheduler::gracefully_stop(): waiting..");
         let transaction_sender = self.transaction_sender.take().unwrap();
         drop(transaction_sender);
-        let executing_thread_cpu_us: usize = self.executing_thread_handles.take().unwrap().iter().map(|executing_thread_handle| {
+        let executing_thread_cpu_us: Result<Vec<_>, _> = self.executing_thread_handles.take().unwrap().iter().map(|executing_thread_handle| {
             executing_thread_handle.join().unwrap().map(|u| u.as_micros())
         }).collect();
         let h = self.scheduler_thread_handle.take().unwrap();

From 6e0f5ed78a4339f38c9ff09f83a0ba71b055e2f8 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 12 Sep 2022 13:26:56 +0900
Subject: [PATCH 2471/3199] save

---
 runtime/src/bank.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index b6b18587ff9b1a..aab962b9c9aa0e 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1411,7 +1411,7 @@ impl Scheduler {
         info!("Scheduler::gracefully_stop(): waiting..");
         let transaction_sender = self.transaction_sender.take().unwrap();
         drop(transaction_sender);
-        let executing_thread_cpu_us: Result<Vec<_>, _> = self.executing_thread_handles.take().unwrap().iter().map(|executing_thread_handle| {
+        let executing_thread_cpu_us: Result<Vec<_>> = self.executing_thread_handles.take().unwrap().iter().map(|executing_thread_handle| {
             executing_thread_handle.join().unwrap().map(|u| u.as_micros())
         }).collect();
         let h = self.scheduler_thread_handle.take().unwrap();

From a62cef114f249f295cb174f844a3725ef5c00c41 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 12 Sep 2022 13:27:06 +0900
Subject: [PATCH 2472/3199] save

---
 runtime/src/bank.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index aab962b9c9aa0e..3995708a809087 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1413,7 +1413,7 @@ impl Scheduler {
         drop(transaction_sender);
         let executing_thread_cpu_us: Result<Vec<_>> = self.executing_thread_handles.take().unwrap().iter().map(|executing_thread_handle| {
             executing_thread_handle.join().unwrap().map(|u| u.as_micros())
-        }).collect();
+        }).collect()?;
         let h = self.scheduler_thread_handle.take().unwrap();
         let scheduler_thread_cpu_us = h.join().unwrap()?.as_micros();
         let h = self.error_collector_thread_handle.take().unwrap();

From 1d3f09b0521abaf2e4cb9e120417b8dd65780189 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 12 Sep 2022 13:27:20 +0900
Subject: [PATCH 2473/3199] save

---
 runtime/src/bank.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 3995708a809087..aab962b9c9aa0e 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1413,7 +1413,7 @@ impl Scheduler {
         drop(transaction_sender);
         let executing_thread_cpu_us: Result<Vec<_>> = self.executing_thread_handles.take().unwrap().iter().map(|executing_thread_handle| {
             executing_thread_handle.join().unwrap().map(|u| u.as_micros())
-        }).collect()?;
+        }).collect();
         let h = self.scheduler_thread_handle.take().unwrap();
         let scheduler_thread_cpu_us = h.join().unwrap()?.as_micros();
         let h = self.error_collector_thread_handle.take().unwrap();

From 53ed35ffaa3737f735ff9a645246fbf8fc034e64 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 12 Sep 2022 13:27:54 +0900
Subject: [PATCH 2474/3199] save

---
 runtime/src/bank.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index aab962b9c9aa0e..58c3aff73593ab 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1411,7 +1411,7 @@ impl Scheduler {
         info!("Scheduler::gracefully_stop(): waiting..");
         let transaction_sender = self.transaction_sender.take().unwrap();
         drop(transaction_sender);
-        let executing_thread_cpu_us: Result<Vec<_>> = self.executing_thread_handles.take().unwrap().iter().map(|executing_thread_handle| {
+        let executing_thread_cpu_us: Result<Vec<usize>> = self.executing_thread_handles.take().unwrap().iter().map(|executing_thread_handle| {
             executing_thread_handle.join().unwrap().map(|u| u.as_micros())
         }).collect();
         let h = self.scheduler_thread_handle.take().unwrap();

From f6794e8bd078d160a169d95cda3185934298aae7 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 12 Sep 2022 13:28:12 +0900
Subject: [PATCH 2475/3199] save

---
 runtime/src/bank.rs | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 58c3aff73593ab..daf2f5dc2876cf 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1411,9 +1411,10 @@ impl Scheduler {
         info!("Scheduler::gracefully_stop(): waiting..");
         let transaction_sender = self.transaction_sender.take().unwrap();
         drop(transaction_sender);
-        let executing_thread_cpu_us: Result<Vec<usize>> = self.executing_thread_handles.take().unwrap().iter().map(|executing_thread_handle| {
+        let executing_thread_cpu_us: Result<Vec<>> = self.executing_thread_handles.take().unwrap().iter().map(|executing_thread_handle| {
             executing_thread_handle.join().unwrap().map(|u| u.as_micros())
         }).collect();
+        executing_thread_cpu_us?;
         let h = self.scheduler_thread_handle.take().unwrap();
         let scheduler_thread_cpu_us = h.join().unwrap()?.as_micros();
         let h = self.error_collector_thread_handle.take().unwrap();

From 48579dd17aa992e52763c3f60a61d298af15cc41 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 12 Sep 2022 13:28:38 +0900
Subject: [PATCH 2476/3199] save

---
 runtime/src/bank.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index daf2f5dc2876cf..6772103fea74dc 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1411,10 +1411,10 @@ impl Scheduler {
         info!("Scheduler::gracefully_stop(): waiting..");
         let transaction_sender = self.transaction_sender.take().unwrap();
         drop(transaction_sender);
-        let executing_thread_cpu_us: Result<Vec<>> = self.executing_thread_handles.take().unwrap().iter().map(|executing_thread_handle| {
+        let executing_thread_cpu_us: Result<Vec<_>> = self.executing_thread_handles.take().unwrap().iter().map(|executing_thread_handle| {
             executing_thread_handle.join().unwrap().map(|u| u.as_micros())
         }).collect();
-        executing_thread_cpu_us?;
+        let executing_thread_cpu_us = executing_thread_cpu_us?;
         let h = self.scheduler_thread_handle.take().unwrap();
         let scheduler_thread_cpu_us = h.join().unwrap()?.as_micros();
         let h = self.error_collector_thread_handle.take().unwrap();

From abe265974c9189fa0869b7c6ee975a7beec3c182 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 12 Sep 2022 13:28:56 +0900
Subject: [PATCH 2477/3199] save

---
 runtime/src/bank.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 6772103fea74dc..414d418fe8373f 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1420,7 +1420,7 @@ impl Scheduler {
         let h = self.error_collector_thread_handle.take().unwrap();
         let error_collector_thread_cpu_us = h.join().unwrap()?.as_micros();
 
-        info!("Scheduler::gracefully_stop(): stopped: schduler: {}, error_collector: {}, executing: {} = {:?}", scheduler_thread_cpu_us, error_collector_thread_cpu_us, executing_thread_cpu_us.iter().sum(), &executing_thread_cpu_us);
+        info!("Scheduler::gracefully_stop(): stopped: schduler: {}, error_collector: {}, executing: {} = {:?}", scheduler_thread_cpu_us, error_collector_thread_cpu_us, executing_thread_cpu_us.iter().sum::<u128>(), &executing_thread_cpu_us);
 
         Ok(())
     }

From 3a9c088bc7f28ce973ca27ce7cb3b34028a74e2e Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 12 Sep 2022 13:29:49 +0900
Subject: [PATCH 2478/3199] save

---
 runtime/src/bank.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 414d418fe8373f..6e9c6491deb03e 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1411,7 +1411,7 @@ impl Scheduler {
         info!("Scheduler::gracefully_stop(): waiting..");
         let transaction_sender = self.transaction_sender.take().unwrap();
         drop(transaction_sender);
-        let executing_thread_cpu_us: Result<Vec<_>> = self.executing_thread_handles.take().unwrap().iter().map(|executing_thread_handle| {
+        let executing_thread_cpu_us: Result<Vec<_>> = self.executing_thread_handles.take().unwrap().into_iter().map(|executing_thread_handle| {
             executing_thread_handle.join().unwrap().map(|u| u.as_micros())
         }).collect();
         let executing_thread_cpu_us = executing_thread_cpu_us?;

From 6694e5b721b7f3ee93a82345ae4aad3d80e3fc8d Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 12 Sep 2022 13:30:50 +0900
Subject: [PATCH 2479/3199] save

---
 runtime/src/bank.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 6e9c6491deb03e..ba0a8442f1815e 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1420,7 +1420,7 @@ impl Scheduler {
         let h = self.error_collector_thread_handle.take().unwrap();
         let error_collector_thread_cpu_us = h.join().unwrap()?.as_micros();
 
-        info!("Scheduler::gracefully_stop(): stopped: schduler: {}, error_collector: {}, executing: {} = {:?}", scheduler_thread_cpu_us, error_collector_thread_cpu_us, executing_thread_cpu_us.iter().sum::<u128>(), &executing_thread_cpu_us);
+        info!("Scheduler::gracefully_stop(): stopped: schduler: {}, error_collector: {}, lanes: total({}) = ({:?})", scheduler_thread_cpu_us, error_collector_thread_cpu_us, executing_thread_cpu_us.iter().sum::<u128>(), &executing_thread_cpu_us);
 
         Ok(())
     }

From ebaa8b1d440a8ffd2cf1757955227d7430aad2dd Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 12 Sep 2022 13:33:22 +0900
Subject: [PATCH 2480/3199] save

---
 runtime/src/bank.rs | 12 +++++++++---
 1 file changed, 9 insertions(+), 3 deletions(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index ba0a8442f1815e..8c44f9b45df0fb 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1253,12 +1253,14 @@ impl Default for Scheduler {
 
         let send_metrics = std::env::var("SEND_METRICS").is_ok();
 
+        let max_thread_priority = std::env::var("MAX_THREAD_PRIORITY").is_ok();
+
         let executing_thread_handles = (0..executing_thread_count).map(|thx| {
             let (scheduled_ee_receiver, processed_ee_sender) = (scheduled_ee_receiver.clone(), processed_ee_sender.clone());
             let bank = bank.clone();
 
             std::thread::Builder::new().name(format!("solScExLane{:02}", thx)).spawn(move || {
-            let started =  cpu_time::ThreadTime::now();
+            let started = cpu_time::ThreadTime::now();
             let current_thread_name = std::thread::current().name().unwrap().to_string();
 
             while let Ok(solana_scheduler::ExecutablePayload(mut ee)) = scheduled_ee_receiver.recv() {
@@ -1343,7 +1345,7 @@ impl Default for Scheduler {
         let errors_in_collector_thread = Arc::clone(&errors);
 
         let error_collector_thread_handle = std::thread::Builder::new().name(format!("solScErrCol{:02}", 0)).spawn(move || {
-            let started =  cpu_time::ThreadTime::now();
+            let started = cpu_time::ThreadTime::now();
 
             while let Ok(solana_scheduler::ExaminablePayload(mut ee)) = retired_ee_receiver.recv() {
                 if ee.is_aborted() {
@@ -1360,7 +1362,11 @@ impl Default for Scheduler {
         }).unwrap();
 
         let scheduler_thread_handle = std::thread::Builder::new().name("solScheduler".to_string()).spawn(move || {
-            let started =  cpu_time::ThreadTime::now();
+            let started = cpu_time::ThreadTime::now();
+            if max_thread_priority {
+                thread_priority::set_current_thread_priority(ThreadPriority::Max).unwrap();
+            }
+
             let mut runnable_queue = solana_scheduler::TaskQueue::default();
             let max_executing_queue_count = std::env::var("MAX_EXECUTING_QUEUE_COUNT")
                 .unwrap_or(format!("{}", 1))

From bdc0a0a73fa1ec35778e3cad09b5be5800e1ff7b Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 12 Sep 2022 13:33:37 +0900
Subject: [PATCH 2481/3199] save

---
 runtime/src/bank.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 8c44f9b45df0fb..ae9cb1a8843787 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1364,7 +1364,7 @@ impl Default for Scheduler {
         let scheduler_thread_handle = std::thread::Builder::new().name("solScheduler".to_string()).spawn(move || {
             let started = cpu_time::ThreadTime::now();
             if max_thread_priority {
-                thread_priority::set_current_thread_priority(ThreadPriority::Max).unwrap();
+                thread_priority::set_current_thread_priority(thread_prioirty::ThreadPriority::Max).unwrap();
             }
 
             let mut runnable_queue = solana_scheduler::TaskQueue::default();

From 989040f8555911330b49c61bebb5b6601e19d078 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 12 Sep 2022 13:33:50 +0900
Subject: [PATCH 2482/3199] save

---
 runtime/src/bank.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index ae9cb1a8843787..2086eadf21d21f 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1364,7 +1364,7 @@ impl Default for Scheduler {
         let scheduler_thread_handle = std::thread::Builder::new().name("solScheduler".to_string()).spawn(move || {
             let started = cpu_time::ThreadTime::now();
             if max_thread_priority {
-                thread_priority::set_current_thread_priority(thread_prioirty::ThreadPriority::Max).unwrap();
+                thread_priority::set_current_thread_priority(thread_priority::ThreadPriority::Max).unwrap();
             }
 
             let mut runnable_queue = solana_scheduler::TaskQueue::default();

From 62eb7db237df695d41baaf7a8969a585f3db260c Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 12 Sep 2022 13:34:50 +0900
Subject: [PATCH 2483/3199] save

---
 runtime/src/bank.rs | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 2086eadf21d21f..013603c8d1dd15 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1261,6 +1261,10 @@ impl Default for Scheduler {
 
             std::thread::Builder::new().name(format!("solScExLane{:02}", thx)).spawn(move || {
             let started = cpu_time::ThreadTime::now();
+            if max_thread_priority {
+                thread_priority::set_current_thread_priority(thread_priority::ThreadPriority::Max).unwrap();
+            }
+
             let current_thread_name = std::thread::current().name().unwrap().to_string();
 
             while let Ok(solana_scheduler::ExecutablePayload(mut ee)) = scheduled_ee_receiver.recv() {
@@ -1346,6 +1350,9 @@ impl Default for Scheduler {
 
         let error_collector_thread_handle = std::thread::Builder::new().name(format!("solScErrCol{:02}", 0)).spawn(move || {
             let started = cpu_time::ThreadTime::now();
+            if max_thread_priority {
+                thread_priority::set_current_thread_priority(thread_priority::ThreadPriority::Max).unwrap();
+            }
 
             while let Ok(solana_scheduler::ExaminablePayload(mut ee)) = retired_ee_receiver.recv() {
                 if ee.is_aborted() {

From 4409c1a56d340cb1d037bae2dbf954429703f93e Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 12 Sep 2022 13:37:43 +0900
Subject: [PATCH 2484/3199] save

---
 runtime/src/bank.rs  | 9 +++++++--
 scheduler/src/lib.rs | 4 +++-
 2 files changed, 10 insertions(+), 3 deletions(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 013603c8d1dd15..b1021b36350799 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1198,6 +1198,7 @@ impl AbiExample for BuiltinPrograms {
 
 #[derive(Debug)]
 struct Scheduler {
+    random_id: u64,
     scheduler_thread_handle: Option<std::thread::JoinHandle<Result<Duration>>>,
     executing_thread_handles: Option<Vec<std::thread::JoinHandle<Result<Duration>>>>,
     error_collector_thread_handle: Option<std::thread::JoinHandle<Result<Duration>>>,
@@ -1368,6 +1369,8 @@ impl Default for Scheduler {
             Ok(started.elapsed())
         }).unwrap();
 
+        let random_id = rand::thread_rng().gen::<u64>();
+
         let scheduler_thread_handle = std::thread::Builder::new().name("solScheduler".to_string()).spawn(move || {
             let started = cpu_time::ThreadTime::now();
             if max_thread_priority {
@@ -1381,6 +1384,7 @@ impl Default for Scheduler {
                 .unwrap();
 
             solana_scheduler::ScheduleStage::run(
+                random_id,
                 max_executing_queue_count,
                 &mut runnable_queue,
                 &mut address_book,
@@ -1397,6 +1401,7 @@ impl Default for Scheduler {
         }).unwrap();
 
         let s = Self {
+            random_id,
             scheduler_thread_handle: Some(scheduler_thread_handle),
             executing_thread_handles: Some(executing_thread_handles),
             error_collector_thread_handle: Some(error_collector_thread_handle),
@@ -1407,7 +1412,7 @@ impl Default for Scheduler {
             bank,
             slot: Default::default(),
         };
-        info!("scheduler: setup done with {}us", start.elapsed().as_micros());
+        info!("scheduler: id_{:016x} setup done with {}us", random_id, start.elapsed().as_micros());
 
         s
     }
@@ -1433,7 +1438,7 @@ impl Scheduler {
         let h = self.error_collector_thread_handle.take().unwrap();
         let error_collector_thread_cpu_us = h.join().unwrap()?.as_micros();
 
-        info!("Scheduler::gracefully_stop(): stopped: schduler: {}, error_collector: {}, lanes: total({}) = ({:?})", scheduler_thread_cpu_us, error_collector_thread_cpu_us, executing_thread_cpu_us.iter().sum::<u128>(), &executing_thread_cpu_us);
+        info!("Scheduler::gracefully_stop(): id_{:016x} stopped: schduler: {}, error_collector: {}, lanes: total({}) = ({:?})", self.random_id, scheduler_thread_cpu_us, error_collector_thread_cpu_us, executing_thread_cpu_us.iter().sum::<u128>(), &executing_thread_cpu_us);
 
         Ok(())
     }
diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 65dfb06d2c4c93..4d2f2cd687ff7e 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1417,6 +1417,7 @@ impl ScheduleStage {
 
     fn _run<'a, AST: AtScheduleThread>(
         ast: AST,
+        random_id: u64,
         max_executing_queue_count: usize,
         runnable_queue: &mut TaskQueue,
         address_book: &mut AddressBook,
@@ -1428,7 +1429,6 @@ impl ScheduleStage {
     ) {
         let start_time = std::time::Instant::now();
         let (mut last_time, mut last_processed_count) = (start_time.clone(), 0_usize);
-        let random_id = rand::thread_rng().gen::<u64>();
         info!("schedule_once:initial id_{:016x}", random_id);
 
         let mut executing_queue_count = 0_usize;
@@ -1694,6 +1694,7 @@ impl ScheduleStage {
     }
 
     pub fn run(
+        random_id: u64,
         max_executing_queue_count: usize,
         runnable_queue: &mut TaskQueue,
         address_book: &mut AddressBook,
@@ -1708,6 +1709,7 @@ impl ScheduleStage {
 
         Self::_run::<AtTopOfScheduleThread>(
             AtTopOfScheduleThread,
+            random_id,
             max_executing_queue_count,
             runnable_queue,
             address_book,

From 008192ea18a5cf822d09c8ba82e56f7f8626c0ea Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 12 Sep 2022 13:39:41 +0900
Subject: [PATCH 2485/3199] save

---
 scheduler/src/lib.rs | 14 ++++----------
 1 file changed, 4 insertions(+), 10 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 4d2f2cd687ff7e..cc5c98d4693ba2 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -63,7 +63,7 @@ impl ExecutionEnvironment {
             .contention_count
             .load(std::sync::atomic::Ordering::SeqCst)
             > 0;
-        for mut lock_attempt in self.finalized_lock_attempts.iter_mut() {
+        for lock_attempt in self.finalized_lock_attempts.iter_mut() {
             let contended_unique_weights = lock_attempt.target_contended_unique_weights();
             contended_unique_weights
                 .heaviest_task_cursor()
@@ -1007,7 +1007,7 @@ impl ScheduleStage {
         task_selection: &mut TaskSelection,
         failed_lock_count: &mut usize,
     ) -> Option<(UniqueWeight, TaskInQueue, Vec<LockAttempt>)> {
-        if let Some(mut a) = address_book.fulfilled_provisional_task_ids.pop_last() {
+        if let Some(a) = address_book.fulfilled_provisional_task_ids.pop_last() {
             trace!(
                 "expediate pop from provisional queue [rest: {}]",
                 address_book.fulfilled_provisional_task_ids.len()
@@ -1019,7 +1019,7 @@ impl ScheduleStage {
         }
 
         loop {
-            if let Some((task_source, mut next_task)) =
+            if let Some((task_source, next_task)) =
                 Self::select_next_task(runnable_queue, address_book, contended_count, task_selection)
             {
                 let from_runnable = task_source == TaskSource::Runnable;
@@ -1457,7 +1457,7 @@ impl ScheduleStage {
                     unsafe impl NotAtScheduleThread for NotAtTopOfScheduleThread {}
                     let nast = NotAtTopOfScheduleThread;
 
-                    while let Ok(ExaminablePayload(mut a)) = ee_receiver.recv() {
+                    while let Ok(ExaminablePayload(a)) = ee_receiver.recv() {
                         assert!(a.task.lock_attempts_not_mut(nast).is_empty());
                         //assert!(a.task.sequence_time() != usize::max_value());
                         //let lock_attempts = std::mem::take(&mut a.lock_attempts);
@@ -1726,9 +1726,3 @@ pub struct SchedulablePayload(pub TaskInQueue);
 pub struct ExecutablePayload(pub Box<ExecutionEnvironment>);
 pub struct UnlockablePayload(pub Box<ExecutionEnvironment>);
 pub struct ExaminablePayload(pub Box<ExecutionEnvironment>);
-
-struct ExecuteStage {
-    //bank: Bank,
-}
-
-impl ExecuteStage {}

From e0df81419846cd39cecca93c899ef0ddde91c006 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 12 Sep 2022 13:40:57 +0900
Subject: [PATCH 2486/3199] save

---
 ledger-tool/src/main.rs | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/ledger-tool/src/main.rs b/ledger-tool/src/main.rs
index 9fe13087bce6e3..3dba8dbf9b50ef 100644
--- a/ledger-tool/src/main.rs
+++ b/ledger-tool/src/main.rs
@@ -280,8 +280,11 @@ fn output_slot(
     let preloader = address_book.preloader();
     let t1 = std::thread::Builder::new()
         .name("sol-scheduler".to_string())
-        .spawn(move || loop {
+        .spawn(move || {
+            let random_id = rand::thread_rng().gen::<u64>();
+            loop {
             ScheduleStage::run(
+                random_id,
                 lane_count * lane_channel_factor,
                 &mut runnable_queue,
                 &mut address_book,
@@ -290,7 +293,7 @@ fn output_slot(
                 &post_execute_env_receiver,
                 Some(&post_schedule_env_sender),
             );
-        })
+        }})
         .unwrap();
     let handles = (0..lane_count)
         .map(|thx| {

From b013a239566947a8f6023ec1ba1d95f93e4f4621 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 12 Sep 2022 13:41:07 +0900
Subject: [PATCH 2487/3199] save

---
 ledger-tool/src/main.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ledger-tool/src/main.rs b/ledger-tool/src/main.rs
index 3dba8dbf9b50ef..b93df54f49f25d 100644
--- a/ledger-tool/src/main.rs
+++ b/ledger-tool/src/main.rs
@@ -281,7 +281,7 @@ fn output_slot(
     let t1 = std::thread::Builder::new()
         .name("sol-scheduler".to_string())
         .spawn(move || {
-            let random_id = rand::thread_rng().gen::<u64>();
+            let random_id = rng::rand::thread_rng().gen::<u64>();
             loop {
             ScheduleStage::run(
                 random_id,

From d549cdadeb53be8e5e9af313cbc627c6504d8b00 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 12 Sep 2022 13:41:37 +0900
Subject: [PATCH 2488/3199] save

---
 ledger-tool/Cargo.toml  | 1 +
 ledger-tool/src/main.rs | 2 +-
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/ledger-tool/Cargo.toml b/ledger-tool/Cargo.toml
index ae45d4b922d0cd..5acac9440f7101 100644
--- a/ledger-tool/Cargo.toml
+++ b/ledger-tool/Cargo.toml
@@ -19,6 +19,7 @@ dashmap = "4.0.2"
 histogram = "0.6.9"
 itertools = "0.10.3"
 log = { version = "0.4.17" }
+rand = "0.7.0"
 regex = "1"
 serde = { version = "1.0", features = ["derive"] }
 serde_json = "1.0.83"
diff --git a/ledger-tool/src/main.rs b/ledger-tool/src/main.rs
index b93df54f49f25d..3dba8dbf9b50ef 100644
--- a/ledger-tool/src/main.rs
+++ b/ledger-tool/src/main.rs
@@ -281,7 +281,7 @@ fn output_slot(
     let t1 = std::thread::Builder::new()
         .name("sol-scheduler".to_string())
         .spawn(move || {
-            let random_id = rng::rand::thread_rng().gen::<u64>();
+            let random_id = rand::thread_rng().gen::<u64>();
             loop {
             ScheduleStage::run(
                 random_id,

From 28a66b445b0b9ae36b285b6563ebc5a2e0fe1b15 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 12 Sep 2022 13:42:02 +0900
Subject: [PATCH 2489/3199] save

---
 Cargo.lock              | 1 +
 ledger-tool/src/main.rs | 1 +
 2 files changed, 2 insertions(+)

diff --git a/Cargo.lock b/Cargo.lock
index 4d59629f989a50..446959eab55907 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -5547,6 +5547,7 @@ dependencies = [
  "histogram",
  "itertools",
  "log",
+ "rand 0.7.3",
  "regex",
  "serde",
  "serde_json",
diff --git a/ledger-tool/src/main.rs b/ledger-tool/src/main.rs
index 3dba8dbf9b50ef..3034bcab690969 100644
--- a/ledger-tool/src/main.rs
+++ b/ledger-tool/src/main.rs
@@ -281,6 +281,7 @@ fn output_slot(
     let t1 = std::thread::Builder::new()
         .name("sol-scheduler".to_string())
         .spawn(move || {
+            use rand::Rng;
             let random_id = rand::thread_rng().gen::<u64>();
             loop {
             ScheduleStage::run(

From 86215e80a41e6e6a2461b276d5947161bb5afdd5 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 12 Sep 2022 14:01:44 +0900
Subject: [PATCH 2490/3199] save

---
 runtime/src/bank.rs | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index b1021b36350799..71f215c22b42ad 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1205,7 +1205,7 @@ struct Scheduler {
     transaction_sender: Option<crossbeam_channel::Sender<solana_scheduler::SchedulablePayload>>,
     preloader: Arc<solana_scheduler::Preloader>,
     graceful_stop_initiated: bool,
-    errors: Arc<std::sync::Mutex<Vec<Result<()>>>>,
+    collected_errors: Arc<std::sync::Mutex<Vec<Result<()>>>>,
     bank: std::sync::Arc<std::sync::RwLock<std::option::Option<std::sync::Weak<Bank>>>>,
     slot: Option<Slot>,
 }
@@ -1346,8 +1346,8 @@ impl Default for Scheduler {
             Ok(started.elapsed())
         }).unwrap()}).collect();
 
-        let errors = Arc::new(std::sync::Mutex::new(Vec::new()));
-        let errors_in_collector_thread = Arc::clone(&errors);
+        let collected_errors = Arc::new(std::sync::Mutex::new(Vec::new()));
+        let collected_errors_in_collector_thread = Arc::clone(&collected_errors);
 
         let error_collector_thread_handle = std::thread::Builder::new().name(format!("solScErrCol{:02}", 0)).spawn(move || {
             let started = cpu_time::ThreadTime::now();
@@ -1361,7 +1361,7 @@ impl Default for Scheduler {
                         "scheduler: Unexpected validator error: {:?}, transaction: {:?}",
                         ee.execution_result, ee.task.tx.0
                     );
-                    errors_in_collector_thread.lock().unwrap().push(ee.execution_result.take().unwrap());
+                    collected_errors_in_collector_thread.lock().unwrap().push(ee.execution_result.take().unwrap());
                 }
 
                 drop(ee);
@@ -1408,7 +1408,7 @@ impl Default for Scheduler {
             transaction_sender: Some(transaction_sender),
             preloader,
             graceful_stop_initiated: Default::default(),
-            errors,
+            collected_errors,
             bank,
             slot: Default::default(),
         };
@@ -1438,13 +1438,13 @@ impl Scheduler {
         let h = self.error_collector_thread_handle.take().unwrap();
         let error_collector_thread_cpu_us = h.join().unwrap()?.as_micros();
 
-        info!("Scheduler::gracefully_stop(): id_{:016x} stopped: schduler: {}, error_collector: {}, lanes: total({}) = ({:?})", self.random_id, scheduler_thread_cpu_us, error_collector_thread_cpu_us, executing_thread_cpu_us.iter().sum::<u128>(), &executing_thread_cpu_us);
+        info!("Scheduler::gracefully_stop(): id_{:016x} cpu times: schduler: {}us, error_collector: {}us, lanes: {}us = {:?}", self.random_id, scheduler_thread_cpu_us, error_collector_thread_cpu_us, executing_thread_cpu_us.iter().sum::<u128>(), &executing_thread_cpu_us);
 
         Ok(())
     }
 
     fn handle_aborted_executions(&self) -> Vec<Result<()>> {
-        std::mem::take(&mut self.errors.lock().unwrap())
+        std::mem::take(&mut self.collected_errors.lock().unwrap())
     }
 }
 
@@ -4127,7 +4127,7 @@ impl Bank {
         let mut w_blockhash_queue = self.blockhash_queue.write().unwrap();
         let new_scheduler = Scheduler::default();
         if maybe_last_error.is_err() {
-            new_scheduler.errors.lock().unwrap().push(maybe_last_error);
+            new_scheduler.collected_errors.lock().unwrap().push(maybe_last_error);
         }
         *self.scheduler.write().unwrap() = new_scheduler;
 

From 738786ed91ac16bf95a0a03ca919366d744606d2 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 12 Sep 2022 14:05:02 +0900
Subject: [PATCH 2491/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index cc5c98d4693ba2..e1e270c59dbda3 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1690,7 +1690,7 @@ impl ScheduleStage {
 
         let elapsed = start_time.elapsed();
         let elapsed2 = elapsed.as_micros();
-        info!("schedule_once:final id_{:016x} (from_disconnected: {}, from_exec_disconnected: {}, no_more_work: {}) ch(prev: {}, exec: {}|{}), runnnable: {}, contended: {}, (immediate+provisional)/max: ({}+{})/{} uncontended: {} stuck: {} miss: {}, overall: {}txs/{}us={}tps!", random_id, from_disconnected, from_exec_disconnected, no_more_work, from_prev.len(), to_execute_substage.len(), from_exec.len(), runnable_queue.task_count(), contended_count, executing_queue_count, provisioning_tracker_count, max_executing_queue_count, address_book.uncontended_task_ids.len(), address_book.stuck_tasks.len(), failed_lock_count, processed_count, elapsed.as_micros(), 1_000_000_u128*(processed_count as u128)/elapsed2);
+        info!("schedule_once:final id_{:016x} (no_more_work: {}) ch(prev: {}, exec: {}|{}), runnnable: {}, contended: {}, (immediate+provisional)/max: ({}+{})/{} uncontended: {} stuck: {} miss: {}, overall: {}txs/{}us={}tps!", random_id, no_more_work, (if from_disconnected { "-".to_string() } else { format!("{}", from_prev.len()) }), to_execute_substage.len(), (if from_exec_disconnected { "-".to_string() } else { format!("{}", from_exec.len())}), runnable_queue.task_count(), contended_count, executing_queue_count, provisioning_tracker_count, max_executing_queue_count, address_book.uncontended_task_ids.len(), address_book.stuck_tasks.len(), failed_lock_count, processed_count, elapsed.as_micros(), 1_000_000_u128*(processed_count as u128)/elapsed2);
     }
 
     pub fn run(

From 9a792875a00738452a0a7affab62ef3356382d72 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 12 Sep 2022 14:20:56 +0900
Subject: [PATCH 2492/3199] save

---
 ledger/src/blockstore_processor.rs | 1 +
 1 file changed, 1 insertion(+)

diff --git a/ledger/src/blockstore_processor.rs b/ledger/src/blockstore_processor.rs
index 13e36fd2260708..9690aae9792ccf 100644
--- a/ledger/src/blockstore_processor.rs
+++ b/ledger/src/blockstore_processor.rs
@@ -602,6 +602,7 @@ fn process_entries_with_callback(
             EntryType::Transactions(transactions) => {
                 let starting_index = *starting_index;
                 let transaction_indexes = if randomize {
+                    panic!("randomize isn't supported");
                     let mut transactions_and_indexes: Vec<(SanitizedTransaction, usize)> =
                         transactions.drain(..).zip(starting_index..).collect();
                     transactions_and_indexes.shuffle(&mut rng);

From 6c180602e4e7d016412bf6b380514d12275e5cab Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 12 Sep 2022 14:51:19 +0900
Subject: [PATCH 2493/3199] save

---
 runtime/src/bank.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 71f215c22b42ad..089536458cd1a1 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1332,7 +1332,7 @@ impl Default for Scheduler {
                         ("thread", current_thread_name, String),
                         ("signature", &sig, String),
                         ("account_locks_in_json", "{}", String),
-                        ("status", format!("{:?}", status_str.unwrap()), String),
+                        ("status", status_str.unwrap(), String),
                         ("duration", duration_with_overhead, i64),
                         ("compute_units", ee.cu, i64),
                     );

From a58983a0e0e9c5bf91207a3da4c123eb1f2d6c3a Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 12 Sep 2022 16:30:57 +0900
Subject: [PATCH 2494/3199] save

---
 runtime/src/bank.rs | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 089536458cd1a1..0beb66803cf24a 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1272,7 +1272,8 @@ impl Default for Scheduler {
                 let mut process_message_time = Measure::start("process_message_time");
 
                 let current_execute_clock = ee.task.execute_time();
-                trace!("execute_substage: thread: {} transaction_index: {} execute_clock: {}", thx, ee.task.transaction_index_in_entries_for_replay(), current_execute_clock);
+                let transaction_index = ee.task.transaction_index_in_entries_for_replay();
+                trace!("execute_substage: thread: {} transaction_index: {} execute_clock: {}", thx, transaction_index, current_execute_clock);
 
                 let ro_bank = bank.read().unwrap();
                 let weak_bank = ro_bank.as_ref().unwrap().upgrade();
@@ -1287,6 +1288,7 @@ impl Default for Scheduler {
                     TransactionBatch::new(vec![lock_result], &bank, Cow::Owned(vec![ee.task.tx.0.clone()]));
                 batch.set_needs_unlock(false);
 
+                let mut timings = Default::default();
                 let (tx_results, _balances) = bank.load_execute_and_commit_transactions(
                     &batch,
                     MAX_PROCESSING_AGE,
@@ -1294,7 +1296,7 @@ impl Default for Scheduler {
                     false,
                     false,
                     false,
-                    &mut Default::default(),
+                    &mut timings,
                     None
                 );
                 drop(bank);
@@ -1302,6 +1304,8 @@ impl Default for Scheduler {
                 drop(weak_bank);
                 drop(ro_bank);
 
+                trace!("execute_substage: slot: {} transaction_index: {} timings: {}", slot, transaction_index, timings);
+
                 let TransactionResults {
                     fee_collection_results,
                     execution_results,

From 34dd4e2f5491527c6ec25a254e76a84a177fedb6 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 12 Sep 2022 16:31:23 +0900
Subject: [PATCH 2495/3199] save

---
 runtime/src/bank.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 0beb66803cf24a..818c361be9b00c 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1304,7 +1304,7 @@ impl Default for Scheduler {
                 drop(weak_bank);
                 drop(ro_bank);
 
-                trace!("execute_substage: slot: {} transaction_index: {} timings: {}", slot, transaction_index, timings);
+                trace!("execute_substage: slot: {} transaction_index: {} timings: {:?}", slot, transaction_index, timings);
 
                 let TransactionResults {
                     fee_collection_results,

From e5db59ca0c8662cbb26f10262ca28afc43c0873b Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 12 Sep 2022 16:31:56 +0900
Subject: [PATCH 2496/3199] save

---
 runtime/src/bank.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 818c361be9b00c..7b9fa52447b50c 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1273,7 +1273,7 @@ impl Default for Scheduler {
 
                 let current_execute_clock = ee.task.execute_time();
                 let transaction_index = ee.task.transaction_index_in_entries_for_replay();
-                trace!("execute_substage: thread: {} transaction_index: {} execute_clock: {}", thx, transaction_index, current_execute_clock);
+                trace!("execute_substage: transaction_index: {} execute_clock: {} at thread: {}", thx, transaction_index, current_execute_clock);
 
                 let ro_bank = bank.read().unwrap();
                 let weak_bank = ro_bank.as_ref().unwrap().upgrade();

From d26f25d74582d5a3e172297d5eabf6ee6301ef02 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 12 Sep 2022 16:32:53 +0900
Subject: [PATCH 2497/3199] save

---
 runtime/src/bank.rs | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 7b9fa52447b50c..96c161559d7787 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1442,7 +1442,7 @@ impl Scheduler {
         let h = self.error_collector_thread_handle.take().unwrap();
         let error_collector_thread_cpu_us = h.join().unwrap()?.as_micros();
 
-        info!("Scheduler::gracefully_stop(): id_{:016x} cpu times: schduler: {}us, error_collector: {}us, lanes: {}us = {:?}", self.random_id, scheduler_thread_cpu_us, error_collector_thread_cpu_us, executing_thread_cpu_us.iter().sum::<u128>(), &executing_thread_cpu_us);
+        info!("Scheduler::gracefully_stop(): id_{:016x} cpu times: scheduler: {}us, error_collector: {}us, lanes: {}us = {:?}", self.random_id, scheduler_thread_cpu_us, error_collector_thread_cpu_us, executing_thread_cpu_us.iter().sum::<u128>(), &executing_thread_cpu_us);
 
         Ok(())
     }
@@ -4131,6 +4131,7 @@ impl Bank {
         let mut w_blockhash_queue = self.blockhash_queue.write().unwrap();
         let new_scheduler = Scheduler::default();
         if maybe_last_error.is_err() {
+            info!("register_recent_blockhash: carrying over this error: {:?}", maybe_last_error);
             new_scheduler.collected_errors.lock().unwrap().push(maybe_last_error);
         }
         *self.scheduler.write().unwrap() = new_scheduler;

From 9092eb10ae6b171c2581d16f8bb6fd120c7f2bd3 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 12 Sep 2022 21:15:17 +0900
Subject: [PATCH 2498/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index e1e270c59dbda3..e3834745f5fc57 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1548,7 +1548,7 @@ impl ScheduleStage {
             let (mut from_len, mut from_exec_len) = (0, 0);
 
             loop {
-                let mut selection = TaskSelection::OnlyFromContended(if from_disconnected && runnable_queue.has_no_task() { usize::max_value() } else { 2 });
+                let mut selection = TaskSelection::OnlyFromContended(if from_disconnected && runnable_queue.has_no_task() { usize::max_value() } else { usize::max_value() /*2*/ });
                 while selection.should_proceed() && executing_queue_count + provisioning_tracker_count < max_executing_queue_count {
                     let prefer_immediate = true; //provisioning_tracker_count / 4 > executing_queue_count;
 

From 0f60168d4d1874885d60ad80e0e41283f03c6723 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 12 Sep 2022 21:46:12 +0900
Subject: [PATCH 2499/3199] save

---
 runtime/src/bank.rs | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 96c161559d7787..f167b14d1a2750 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1333,6 +1333,7 @@ impl Default for Scheduler {
                     datapoint_info!(
                         "individual_tx_stats",
                         ("slot", slot, i64),
+                        ("index", transaction_index, i64),
                         ("thread", current_thread_name, String),
                         ("signature", &sig, String),
                         ("account_locks_in_json", "{}", String),
@@ -1425,12 +1426,12 @@ impl Default for Scheduler {
 impl Scheduler {
     fn gracefully_stop(&mut self) -> Result<()> {
         if self.graceful_stop_initiated {
-            info!("Scheduler::gracefully_stop(): (skipped..?)");
+            info!("Scheduler::gracefully_stop(): id_{:016x} (skipped..?)", self.random_id);
             return Ok(());
         }
         self.graceful_stop_initiated = true;
 
-        info!("Scheduler::gracefully_stop(): waiting..");
+        info!("Scheduler::gracefully_stop(): id_{:016x} waiting..", self.random_id);
         let transaction_sender = self.transaction_sender.take().unwrap();
         drop(transaction_sender);
         let executing_thread_cpu_us: Result<Vec<_>> = self.executing_thread_handles.take().unwrap().into_iter().map(|executing_thread_handle| {
@@ -1454,9 +1455,9 @@ impl Scheduler {
 
 impl Drop for Scheduler {
     fn drop(&mut self) {
-        info!("Scheduler::drop(): begin..");
+        info!("Scheduler::drop(): id_{:016x} begin..", self.random_id);
         self.gracefully_stop().unwrap();
-        info!("Scheduler::drop(): end...");
+        info!("Scheduler::drop(): id_{:016x} end...", self.random_id);
     }
 }
 

From 9ea299f2acd1f993f1d8079936c3094a8dcec86b Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 13 Sep 2022 15:10:59 +0900
Subject: [PATCH 2500/3199] save

---
 runtime/src/bank.rs  | 2 +-
 scheduler/src/lib.rs | 2 ++
 2 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index f167b14d1a2750..17c7996d2e4363 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -4132,7 +4132,7 @@ impl Bank {
         let mut w_blockhash_queue = self.blockhash_queue.write().unwrap();
         let new_scheduler = Scheduler::default();
         if maybe_last_error.is_err() {
-            info!("register_recent_blockhash: carrying over this error: {:?}", maybe_last_error);
+            warn!("register_recent_blockhash: carrying over this error: {:?}", maybe_last_error);
             new_scheduler.collected_errors.lock().unwrap().push(maybe_last_error);
         }
         *self.scheduler.write().unwrap() = new_scheduler;
diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index e3834745f5fc57..0f6abaa3e948a3 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1574,10 +1574,12 @@ impl ScheduleStage {
                         debug!("schedule_once id_{:016x} [C] ch(prev: {}, exec: {}|{}), r: {}, u/c: {}/{}, (imm+provi)/max: ({}+{})/{} s: {} l(s+f): {}+{}", random_id, (if from_disconnected { "-".to_string() } else { format!("{}", from_prev.len()) }), to_execute_substage.len(), from_exec.len(), runnable_queue.task_count(), address_book.uncontended_task_ids.len(), contended_count, executing_queue_count, provisioning_tracker_count, max_executing_queue_count, address_book.stuck_tasks.len(), processed_count, failed_lock_count);
                         break;
                     }
+                    /*
                     if !from_exec.is_empty() {
                         trace!("abort aggressive contended queue processing due to non-empty from_exec");
                         break;
                     }
+                    */
 
                     interval_count += 1;
                     if interval_count % 100 == 0 {

From be42132bd6c63e24d0e43d6be6c2c8f4a49eb268 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 13 Sep 2022 21:35:39 +0900
Subject: [PATCH 2501/3199] save

---
 runtime/src/bank.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 17c7996d2e4363..ba604246de8385 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1437,7 +1437,7 @@ impl Scheduler {
         let executing_thread_cpu_us: Result<Vec<_>> = self.executing_thread_handles.take().unwrap().into_iter().map(|executing_thread_handle| {
             executing_thread_handle.join().unwrap().map(|u| u.as_micros())
         }).collect();
-        let executing_thread_cpu_us = executing_thread_cpu_us?;
+        let executing_thread_cpu_us = executing_thread_cpu_us?.sort();
         let h = self.scheduler_thread_handle.take().unwrap();
         let scheduler_thread_cpu_us = h.join().unwrap()?.as_micros();
         let h = self.error_collector_thread_handle.take().unwrap();

From 8a14bd430e03f264b1cbcfebbbca2f60033ae8c5 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 13 Sep 2022 21:36:16 +0900
Subject: [PATCH 2502/3199] save

---
 runtime/src/bank.rs | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index ba604246de8385..979cf593f3dfa8 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1437,7 +1437,8 @@ impl Scheduler {
         let executing_thread_cpu_us: Result<Vec<_>> = self.executing_thread_handles.take().unwrap().into_iter().map(|executing_thread_handle| {
             executing_thread_handle.join().unwrap().map(|u| u.as_micros())
         }).collect();
-        let executing_thread_cpu_us = executing_thread_cpu_us?.sort();
+        let mut executing_thread_cpu_us = executing_thread_cpu_us?;
+        executing_thread_cpu_us.sort();
         let h = self.scheduler_thread_handle.take().unwrap();
         let scheduler_thread_cpu_us = h.join().unwrap()?.as_micros();
         let h = self.error_collector_thread_handle.take().unwrap();

From 5a3f1895f812f154eaf6111e86ed1f3e6134f015 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 13 Sep 2022 21:42:02 +0900
Subject: [PATCH 2503/3199] save

---
 runtime/src/bank.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 979cf593f3dfa8..54795f04c5dd71 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1444,7 +1444,7 @@ impl Scheduler {
         let h = self.error_collector_thread_handle.take().unwrap();
         let error_collector_thread_cpu_us = h.join().unwrap()?.as_micros();
 
-        info!("Scheduler::gracefully_stop(): id_{:016x} cpu times: scheduler: {}us, error_collector: {}us, lanes: {}us = {:?}", self.random_id, scheduler_thread_cpu_us, error_collector_thread_cpu_us, executing_thread_cpu_us.iter().sum::<u128>(), &executing_thread_cpu_us);
+        info!("Scheduler::gracefully_stop(): slot: {} id_{:016x} cpu times: scheduler: {}us, error_collector: {}us, lanes: {}us = {:?}", self.slot.map(|s| format!("{}", s)).or_default("-"), self.random_id, scheduler_thread_cpu_us, error_collector_thread_cpu_us, executing_thread_cpu_us.iter().sum::<u128>(), &executing_thread_cpu_us);
 
         Ok(())
     }

From 86bdf5a739751b7a263e61d35750779abb660b55 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 13 Sep 2022 21:42:14 +0900
Subject: [PATCH 2504/3199] save

---
 runtime/src/bank.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 54795f04c5dd71..3cda4a2e121459 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1444,7 +1444,7 @@ impl Scheduler {
         let h = self.error_collector_thread_handle.take().unwrap();
         let error_collector_thread_cpu_us = h.join().unwrap()?.as_micros();
 
-        info!("Scheduler::gracefully_stop(): slot: {} id_{:016x} cpu times: scheduler: {}us, error_collector: {}us, lanes: {}us = {:?}", self.slot.map(|s| format!("{}", s)).or_default("-"), self.random_id, scheduler_thread_cpu_us, error_collector_thread_cpu_us, executing_thread_cpu_us.iter().sum::<u128>(), &executing_thread_cpu_us);
+        info!("Scheduler::gracefully_stop(): slot: {} id_{:016x} cpu times: scheduler: {}us, error_collector: {}us, lanes: {}us = {:?}", self.slot.map(|s| format!("{}", s)).unwrap_or_default("-"), self.random_id, scheduler_thread_cpu_us, error_collector_thread_cpu_us, executing_thread_cpu_us.iter().sum::<u128>(), &executing_thread_cpu_us);
 
         Ok(())
     }

From 0ffe1b0f16682e1103c7c715b7bb7adc661478bf Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 13 Sep 2022 21:42:32 +0900
Subject: [PATCH 2505/3199] save

---
 runtime/src/bank.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 3cda4a2e121459..cfa4a9d5991e82 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1444,7 +1444,7 @@ impl Scheduler {
         let h = self.error_collector_thread_handle.take().unwrap();
         let error_collector_thread_cpu_us = h.join().unwrap()?.as_micros();
 
-        info!("Scheduler::gracefully_stop(): slot: {} id_{:016x} cpu times: scheduler: {}us, error_collector: {}us, lanes: {}us = {:?}", self.slot.map(|s| format!("{}", s)).unwrap_or_default("-"), self.random_id, scheduler_thread_cpu_us, error_collector_thread_cpu_us, executing_thread_cpu_us.iter().sum::<u128>(), &executing_thread_cpu_us);
+        info!("Scheduler::gracefully_stop(): slot: {} id_{:016x} cpu times: scheduler: {}us, error_collector: {}us, lanes: {}us = {:?}", self.slot.map(|s| format!("{}", s).into()).unwrap_or_default("-"), self.random_id, scheduler_thread_cpu_us, error_collector_thread_cpu_us, executing_thread_cpu_us.iter().sum::<u128>(), &executing_thread_cpu_us);
 
         Ok(())
     }

From be62b994ace5103dc825a06201530e86a41353da Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 13 Sep 2022 21:42:48 +0900
Subject: [PATCH 2506/3199] save

---
 runtime/src/bank.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index cfa4a9d5991e82..e9c2e600f23b49 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1444,7 +1444,7 @@ impl Scheduler {
         let h = self.error_collector_thread_handle.take().unwrap();
         let error_collector_thread_cpu_us = h.join().unwrap()?.as_micros();
 
-        info!("Scheduler::gracefully_stop(): slot: {} id_{:016x} cpu times: scheduler: {}us, error_collector: {}us, lanes: {}us = {:?}", self.slot.map(|s| format!("{}", s).into()).unwrap_or_default("-"), self.random_id, scheduler_thread_cpu_us, error_collector_thread_cpu_us, executing_thread_cpu_us.iter().sum::<u128>(), &executing_thread_cpu_us);
+        info!("Scheduler::gracefully_stop(): slot: {} id_{:016x} cpu times: scheduler: {}us, error_collector: {}us, lanes: {}us = {:?}", self.slot.map(|s| format!("{}", s).into).unwrap_or("-"), self.random_id, scheduler_thread_cpu_us, error_collector_thread_cpu_us, executing_thread_cpu_us.iter().sum::<u128>(), &executing_thread_cpu_us);
 
         Ok(())
     }

From 929a94bbe75210a97674fa9d5fd605201a50a728 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 13 Sep 2022 21:42:56 +0900
Subject: [PATCH 2507/3199] save

---
 runtime/src/bank.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index e9c2e600f23b49..d3cdffeac43464 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1444,7 +1444,7 @@ impl Scheduler {
         let h = self.error_collector_thread_handle.take().unwrap();
         let error_collector_thread_cpu_us = h.join().unwrap()?.as_micros();
 
-        info!("Scheduler::gracefully_stop(): slot: {} id_{:016x} cpu times: scheduler: {}us, error_collector: {}us, lanes: {}us = {:?}", self.slot.map(|s| format!("{}", s).into).unwrap_or("-"), self.random_id, scheduler_thread_cpu_us, error_collector_thread_cpu_us, executing_thread_cpu_us.iter().sum::<u128>(), &executing_thread_cpu_us);
+        info!("Scheduler::gracefully_stop(): slot: {} id_{:016x} cpu times: scheduler: {}us, error_collector: {}us, lanes: {}us = {:?}", self.slot.map(|s| format!("{}", s)).unwrap_or("-"), self.random_id, scheduler_thread_cpu_us, error_collector_thread_cpu_us, executing_thread_cpu_us.iter().sum::<u128>(), &executing_thread_cpu_us);
 
         Ok(())
     }

From 91996af00e87fbe10040e17d3c7d5bf716fd5aed Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 13 Sep 2022 21:43:23 +0900
Subject: [PATCH 2508/3199] save

---
 runtime/src/bank.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index d3cdffeac43464..921294dc032855 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1444,7 +1444,7 @@ impl Scheduler {
         let h = self.error_collector_thread_handle.take().unwrap();
         let error_collector_thread_cpu_us = h.join().unwrap()?.as_micros();
 
-        info!("Scheduler::gracefully_stop(): slot: {} id_{:016x} cpu times: scheduler: {}us, error_collector: {}us, lanes: {}us = {:?}", self.slot.map(|s| format!("{}", s)).unwrap_or("-"), self.random_id, scheduler_thread_cpu_us, error_collector_thread_cpu_us, executing_thread_cpu_us.iter().sum::<u128>(), &executing_thread_cpu_us);
+        info!("Scheduler::gracefully_stop(): slot: {} id_{:016x} cpu times: scheduler: {}us, error_collector: {}us, lanes: {}us = {:?}", self.slot.map(|s| format!("{}", s)).unwrap_or("-".into()), self.random_id, scheduler_thread_cpu_us, error_collector_thread_cpu_us, executing_thread_cpu_us.iter().sum::<u128>(), &executing_thread_cpu_us);
 
         Ok(())
     }

From 180ab604d2cc9d150d54a6069cb7007fd2a82371 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 13 Sep 2022 21:46:48 +0900
Subject: [PATCH 2509/3199] save

---
 runtime/src/bank.rs | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 921294dc032855..351423a2e0aba5 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1269,7 +1269,7 @@ impl Default for Scheduler {
             let current_thread_name = std::thread::current().name().unwrap().to_string();
 
             while let Ok(solana_scheduler::ExecutablePayload(mut ee)) = scheduled_ee_receiver.recv() {
-                let mut process_message_time = Measure::start("process_message_time");
+                let mut (wall_time, cpu_time) = (Measure::start("process_message_time"), cpu_time::ThreadTime::now());
 
                 let current_execute_clock = ee.task.execute_time();
                 let transaction_index = ee.task.transaction_index_in_entries_for_replay();
@@ -1327,8 +1327,8 @@ impl Default for Scheduler {
                 if send_metrics {
                     let sig = ee.task.tx.0.signature().to_string();
 
-                    process_message_time.stop();
-                    let duration_with_overhead = process_message_time.as_us();
+                    wall_time.stop();
+                    let duration_with_overhead = wall_time.as_us();
 
                     datapoint_info!(
                         "individual_tx_stats",
@@ -1339,6 +1339,7 @@ impl Default for Scheduler {
                         ("account_locks_in_json", "{}", String),
                         ("status", status_str.unwrap(), String),
                         ("duration", duration_with_overhead, i64),
+                        ("cpu_duration", cpu_time.elapsed().as_us(), i64),
                         ("compute_units", ee.cu, i64),
                     );
                 }

From 81ae44489373a484237680601fe5778bde44bed5 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 13 Sep 2022 21:47:02 +0900
Subject: [PATCH 2510/3199] save

---
 runtime/src/bank.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 351423a2e0aba5..9aa8d5e2fe1148 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1269,7 +1269,7 @@ impl Default for Scheduler {
             let current_thread_name = std::thread::current().name().unwrap().to_string();
 
             while let Ok(solana_scheduler::ExecutablePayload(mut ee)) = scheduled_ee_receiver.recv() {
-                let mut (wall_time, cpu_time) = (Measure::start("process_message_time"), cpu_time::ThreadTime::now());
+                let (mut wall_time, mut cpu_time) = (Measure::start("process_message_time"), cpu_time::ThreadTime::now());
 
                 let current_execute_clock = ee.task.execute_time();
                 let transaction_index = ee.task.transaction_index_in_entries_for_replay();

From 00610a234b7501423919ebaa9b265b4e2d3083ac Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 13 Sep 2022 21:47:15 +0900
Subject: [PATCH 2511/3199] save

---
 runtime/src/bank.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 9aa8d5e2fe1148..165aa15fab822d 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1328,7 +1328,7 @@ impl Default for Scheduler {
                     let sig = ee.task.tx.0.signature().to_string();
 
                     wall_time.stop();
-                    let duration_with_overhead = wall_time.as_us();
+                    let duration_with_overhead = wall_time.as_micros();
 
                     datapoint_info!(
                         "individual_tx_stats",

From 65c08e8bd899b2544da4bb9c16b4ae52b6393d12 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 13 Sep 2022 21:47:33 +0900
Subject: [PATCH 2512/3199] save

---
 runtime/src/bank.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 165aa15fab822d..a5d8f17135b7cd 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1328,7 +1328,7 @@ impl Default for Scheduler {
                     let sig = ee.task.tx.0.signature().to_string();
 
                     wall_time.stop();
-                    let duration_with_overhead = wall_time.as_micros();
+                    let duration_with_overhead = wall_time.as_us();
 
                     datapoint_info!(
                         "individual_tx_stats",
@@ -1339,7 +1339,7 @@ impl Default for Scheduler {
                         ("account_locks_in_json", "{}", String),
                         ("status", status_str.unwrap(), String),
                         ("duration", duration_with_overhead, i64),
-                        ("cpu_duration", cpu_time.elapsed().as_us(), i64),
+                        ("cpu_duration", cpu_time.elapsed().as_micros(), i64),
                         ("compute_units", ee.cu, i64),
                     );
                 }

From 33f5da9d9633121a352d56f5ba84413eb571afb8 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 13 Sep 2022 21:47:54 +0900
Subject: [PATCH 2513/3199] save

---
 runtime/src/bank.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index a5d8f17135b7cd..2b1f5e1226f3ff 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1269,7 +1269,7 @@ impl Default for Scheduler {
             let current_thread_name = std::thread::current().name().unwrap().to_string();
 
             while let Ok(solana_scheduler::ExecutablePayload(mut ee)) = scheduled_ee_receiver.recv() {
-                let (mut wall_time, mut cpu_time) = (Measure::start("process_message_time"), cpu_time::ThreadTime::now());
+                let (mut wall_time, cpu_time) = (Measure::start("process_message_time"), cpu_time::ThreadTime::now());
 
                 let current_execute_clock = ee.task.execute_time();
                 let transaction_index = ee.task.transaction_index_in_entries_for_replay();

From 8259a21a494849eeb8765568e17eead123ca9970 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 13 Sep 2022 21:49:02 +0900
Subject: [PATCH 2514/3199] save

---
 runtime/src/bank.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 2b1f5e1226f3ff..348ae38573cd49 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1261,7 +1261,7 @@ impl Default for Scheduler {
             let bank = bank.clone();
 
             std::thread::Builder::new().name(format!("solScExLane{:02}", thx)).spawn(move || {
-            let started = cpu_time::ThreadTime::now();
+            let started = (std::time::Instant::now(), cpu_time::ThreadTime::now());
             if max_thread_priority {
                 thread_priority::set_current_thread_priority(thread_priority::ThreadPriority::Max).unwrap();
             }
@@ -1349,7 +1349,7 @@ impl Default for Scheduler {
                 //ee.reindex_with_address_book();
                 processed_ee_sender.send(solana_scheduler::UnlockablePayload(ee)).unwrap();
             }
-            Ok(started.elapsed())
+            Ok((started.0.elapsed(), started.1.elapsed()))
         }).unwrap()}).collect();
 
         let collected_errors = Arc::new(std::sync::Mutex::new(Vec::new()));

From 5cae160be47e2a54af80ea0bc59a8c1b64756183 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 13 Sep 2022 21:49:33 +0900
Subject: [PATCH 2515/3199] save

---
 runtime/src/bank.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 348ae38573cd49..4a0400f2457e18 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1199,8 +1199,8 @@ impl AbiExample for BuiltinPrograms {
 #[derive(Debug)]
 struct Scheduler {
     random_id: u64,
-    scheduler_thread_handle: Option<std::thread::JoinHandle<Result<Duration>>>,
-    executing_thread_handles: Option<Vec<std::thread::JoinHandle<Result<Duration>>>>,
+    scheduler_thread_handle: Option<std::thread::JoinHandle<Result<(Duration>>>,
+    executing_thread_handles: Option<Vec<std::thread::JoinHandle<Result<(Duration, Duration)>>>>,
     error_collector_thread_handle: Option<std::thread::JoinHandle<Result<Duration>>>,
     transaction_sender: Option<crossbeam_channel::Sender<solana_scheduler::SchedulablePayload>>,
     preloader: Arc<solana_scheduler::Preloader>,

From 464ff848a15caa874dd8959005b66ad5c3b828a6 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 13 Sep 2022 21:49:42 +0900
Subject: [PATCH 2516/3199] save

---
 runtime/src/bank.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 4a0400f2457e18..c5d408edd56596 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1199,7 +1199,7 @@ impl AbiExample for BuiltinPrograms {
 #[derive(Debug)]
 struct Scheduler {
     random_id: u64,
-    scheduler_thread_handle: Option<std::thread::JoinHandle<Result<(Duration>>>,
+    scheduler_thread_handle: Option<std::thread::JoinHandle<Result<Duration>>>,
     executing_thread_handles: Option<Vec<std::thread::JoinHandle<Result<(Duration, Duration)>>>>,
     error_collector_thread_handle: Option<std::thread::JoinHandle<Result<Duration>>>,
     transaction_sender: Option<crossbeam_channel::Sender<solana_scheduler::SchedulablePayload>>,

From a18c2e13413882f816af8aa606a50a0f34fa090d Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 13 Sep 2022 21:50:22 +0900
Subject: [PATCH 2517/3199] save

---
 runtime/src/bank.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index c5d408edd56596..5ba59375c1fa41 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1435,7 +1435,7 @@ impl Scheduler {
         info!("Scheduler::gracefully_stop(): id_{:016x} waiting..", self.random_id);
         let transaction_sender = self.transaction_sender.take().unwrap();
         drop(transaction_sender);
-        let executing_thread_cpu_us: Result<Vec<_>> = self.executing_thread_handles.take().unwrap().into_iter().map(|executing_thread_handle| {
+        let executing_thread_cpu_us: Result<Vec<_>> = self.executing_thread_handles.take().unwrap().into_iter().unzip().map(|executing_thread_handle| {
             executing_thread_handle.join().unwrap().map(|u| u.as_micros())
         }).collect();
         let mut executing_thread_cpu_us = executing_thread_cpu_us?;

From 2d445d47c6ba386ccba38da861dd22ef992190d4 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 13 Sep 2022 21:51:37 +0900
Subject: [PATCH 2518/3199] save

---
 runtime/src/bank.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 5ba59375c1fa41..00707b6ecc0dd0 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1435,8 +1435,8 @@ impl Scheduler {
         info!("Scheduler::gracefully_stop(): id_{:016x} waiting..", self.random_id);
         let transaction_sender = self.transaction_sender.take().unwrap();
         drop(transaction_sender);
-        let executing_thread_cpu_us: Result<Vec<_>> = self.executing_thread_handles.take().unwrap().into_iter().unzip().map(|executing_thread_handle| {
-            executing_thread_handle.join().unwrap().map(|u| u.as_micros())
+        let executing_thread_cpu_us: Result<Vec<_>> = self.executing_thread_handles.take().unwrap().into_iter().map(|executing_thread_handle| {
+            executing_thread_handle.join().unwrap().map(|u| (u.0.as_micros(), u.1.as_micros()))
         }).collect();
         let mut executing_thread_cpu_us = executing_thread_cpu_us?;
         executing_thread_cpu_us.sort();

From 87cb3c0ca88d73f9099300f4183233717f2dd9a3 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 13 Sep 2022 21:52:03 +0900
Subject: [PATCH 2519/3199] save

---
 runtime/src/bank.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 00707b6ecc0dd0..6a66a89d5aa516 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1438,7 +1438,7 @@ impl Scheduler {
         let executing_thread_cpu_us: Result<Vec<_>> = self.executing_thread_handles.take().unwrap().into_iter().map(|executing_thread_handle| {
             executing_thread_handle.join().unwrap().map(|u| (u.0.as_micros(), u.1.as_micros()))
         }).collect();
-        let mut executing_thread_cpu_us = executing_thread_cpu_us?;
+        let mut executing_thread_cpu_us = executing_thread_cpu_us?.unzip();
         executing_thread_cpu_us.sort();
         let h = self.scheduler_thread_handle.take().unwrap();
         let scheduler_thread_cpu_us = h.join().unwrap()?.as_micros();

From a5dbd57a906d627c994b42dad053decb259f908a Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 13 Sep 2022 21:52:21 +0900
Subject: [PATCH 2520/3199] save

---
 runtime/src/bank.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 6a66a89d5aa516..d4999f376ef611 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1438,7 +1438,7 @@ impl Scheduler {
         let executing_thread_cpu_us: Result<Vec<_>> = self.executing_thread_handles.take().unwrap().into_iter().map(|executing_thread_handle| {
             executing_thread_handle.join().unwrap().map(|u| (u.0.as_micros(), u.1.as_micros()))
         }).collect();
-        let mut executing_thread_cpu_us = executing_thread_cpu_us?.unzip();
+        let mut executing_thread_cpu_us = executing_thread_cpu_us?.into_iter().unzip();
         executing_thread_cpu_us.sort();
         let h = self.scheduler_thread_handle.take().unwrap();
         let scheduler_thread_cpu_us = h.join().unwrap()?.as_micros();

From 8f51a26bf67e934b85e42c5e13faf04ca2f9bb9e Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 13 Sep 2022 21:53:31 +0900
Subject: [PATCH 2521/3199] save

---
 runtime/src/bank.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index d4999f376ef611..16e2f73e48c191 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1438,7 +1438,7 @@ impl Scheduler {
         let executing_thread_cpu_us: Result<Vec<_>> = self.executing_thread_handles.take().unwrap().into_iter().map(|executing_thread_handle| {
             executing_thread_handle.join().unwrap().map(|u| (u.0.as_micros(), u.1.as_micros()))
         }).collect();
-        let mut executing_thread_cpu_us = executing_thread_cpu_us?.into_iter().unzip();
+        let mut (executing_thread_cpu_us, _) = executing_thread_cpu_us?.into_iter().unzip();
         executing_thread_cpu_us.sort();
         let h = self.scheduler_thread_handle.take().unwrap();
         let scheduler_thread_cpu_us = h.join().unwrap()?.as_micros();

From 0ff6bf794e4dd54e2820a6e27e51030b182bdf02 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 13 Sep 2022 21:53:44 +0900
Subject: [PATCH 2522/3199] save

---
 runtime/src/bank.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 16e2f73e48c191..6c6162b689062d 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1438,7 +1438,7 @@ impl Scheduler {
         let executing_thread_cpu_us: Result<Vec<_>> = self.executing_thread_handles.take().unwrap().into_iter().map(|executing_thread_handle| {
             executing_thread_handle.join().unwrap().map(|u| (u.0.as_micros(), u.1.as_micros()))
         }).collect();
-        let mut (executing_thread_cpu_us, _) = executing_thread_cpu_us?.into_iter().unzip();
+        let (mut executing_thread_cpu_us, _) = executing_thread_cpu_us?.into_iter().unzip();
         executing_thread_cpu_us.sort();
         let h = self.scheduler_thread_handle.take().unwrap();
         let scheduler_thread_cpu_us = h.join().unwrap()?.as_micros();

From 9f2d887508e35285713907ebc9cb7f17b150da00 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 13 Sep 2022 21:54:04 +0900
Subject: [PATCH 2523/3199] save

---
 runtime/src/bank.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 6c6162b689062d..0151de143f076c 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1438,7 +1438,7 @@ impl Scheduler {
         let executing_thread_cpu_us: Result<Vec<_>> = self.executing_thread_handles.take().unwrap().into_iter().map(|executing_thread_handle| {
             executing_thread_handle.join().unwrap().map(|u| (u.0.as_micros(), u.1.as_micros()))
         }).collect();
-        let (mut executing_thread_cpu_us, _) = executing_thread_cpu_us?.into_iter().unzip();
+        let (mut executing_thread_cpu_us, _): (Vec<_>, Vec<_>) = executing_thread_cpu_us?.into_iter().unzip();
         executing_thread_cpu_us.sort();
         let h = self.scheduler_thread_handle.take().unwrap();
         let scheduler_thread_cpu_us = h.join().unwrap()?.as_micros();

From 8ae7a21df24d5d9517efe841b0309ee35fa66c9c Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 13 Sep 2022 21:55:30 +0900
Subject: [PATCH 2524/3199] save

---
 runtime/src/bank.rs | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 0151de143f076c..a0feae683138f6 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1438,8 +1438,9 @@ impl Scheduler {
         let executing_thread_cpu_us: Result<Vec<_>> = self.executing_thread_handles.take().unwrap().into_iter().map(|executing_thread_handle| {
             executing_thread_handle.join().unwrap().map(|u| (u.0.as_micros(), u.1.as_micros()))
         }).collect();
-        let (mut executing_thread_cpu_us, _): (Vec<_>, Vec<_>) = executing_thread_cpu_us?.into_iter().unzip();
+        let mut executing_thread_cpu_us = executing_thread_cpu_us?;
         executing_thread_cpu_us.sort();
+        let (mut executing_thread_cpu_us, mut executing_thread_wall_time_us): (Vec<_>, Vec<_>) = executing_thread_cpu_us.into_iter().unzip();
         let h = self.scheduler_thread_handle.take().unwrap();
         let scheduler_thread_cpu_us = h.join().unwrap()?.as_micros();
         let h = self.error_collector_thread_handle.take().unwrap();

From 37d34dc4901f595ae3e7f6adbc486f9b99ee8e82 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 13 Sep 2022 21:55:46 +0900
Subject: [PATCH 2525/3199] save

---
 runtime/src/bank.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index a0feae683138f6..7cef8d7152770a 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1440,7 +1440,7 @@ impl Scheduler {
         }).collect();
         let mut executing_thread_cpu_us = executing_thread_cpu_us?;
         executing_thread_cpu_us.sort();
-        let (mut executing_thread_cpu_us, mut executing_thread_wall_time_us): (Vec<_>, Vec<_>) = executing_thread_cpu_us.into_iter().unzip();
+        let (executing_thread_cpu_us, executing_thread_wall_time_us): (Vec<_>, Vec<_>) = executing_thread_cpu_us.into_iter().unzip();
         let h = self.scheduler_thread_handle.take().unwrap();
         let scheduler_thread_cpu_us = h.join().unwrap()?.as_micros();
         let h = self.error_collector_thread_handle.take().unwrap();

From 546ae4ea734b9a80ebf68e25990169dcc4c94b38 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 13 Sep 2022 21:58:31 +0900
Subject: [PATCH 2526/3199] save

---
 runtime/src/bank.rs | 13 ++++++++-----
 1 file changed, 8 insertions(+), 5 deletions(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 7cef8d7152770a..216cb30d056fc2 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1435,16 +1435,19 @@ impl Scheduler {
         info!("Scheduler::gracefully_stop(): id_{:016x} waiting..", self.random_id);
         let transaction_sender = self.transaction_sender.take().unwrap();
         drop(transaction_sender);
-        let executing_thread_cpu_us: Result<Vec<_>> = self.executing_thread_handles.take().unwrap().into_iter().map(|executing_thread_handle| {
+        let executing_thread_duration_pairs: Result<Vec<_>> = self.executing_thread_handles.take().unwrap().into_iter().map(|executing_thread_handle| {
             executing_thread_handle.join().unwrap().map(|u| (u.0.as_micros(), u.1.as_micros()))
         }).collect();
-        let mut executing_thread_cpu_us = executing_thread_cpu_us?;
-        executing_thread_cpu_us.sort();
+        let mut executing_thread_duration_pairs = executing_thread_duration_pairs?;
+        executing_thread_duration_pairs.sort();
         let (executing_thread_cpu_us, executing_thread_wall_time_us): (Vec<_>, Vec<_>) = executing_thread_cpu_us.into_iter().unzip();
+
         let h = self.scheduler_thread_handle.take().unwrap();
-        let scheduler_thread_cpu_us = h.join().unwrap()?.as_micros();
+        let scheduler_thread_duration_pairs = h.join().unwrap()?;
+        let (scheduler_thread_cpu_us, scheduler_thread_wall_time_us) = (scheduler_thread_duration_pairs.0.as_micros(), scheduler_thread_duration_pairs.0.as_micros());
         let h = self.error_collector_thread_handle.take().unwrap();
-        let error_collector_thread_cpu_us = h.join().unwrap()?.as_micros();
+        let error_collector_thread_duration_pairs = h.join().unwrap()?
+            .as_micros();
 
         info!("Scheduler::gracefully_stop(): slot: {} id_{:016x} cpu times: scheduler: {}us, error_collector: {}us, lanes: {}us = {:?}", self.slot.map(|s| format!("{}", s)).unwrap_or("-".into()), self.random_id, scheduler_thread_cpu_us, error_collector_thread_cpu_us, executing_thread_cpu_us.iter().sum::<u128>(), &executing_thread_cpu_us);
 

From bf93909688a37c8ba226e02ca235030d18e52e6d Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 13 Sep 2022 21:58:53 +0900
Subject: [PATCH 2527/3199] save

---
 runtime/src/bank.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 216cb30d056fc2..a2b144a7530a28 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1440,7 +1440,7 @@ impl Scheduler {
         }).collect();
         let mut executing_thread_duration_pairs = executing_thread_duration_pairs?;
         executing_thread_duration_pairs.sort();
-        let (executing_thread_cpu_us, executing_thread_wall_time_us): (Vec<_>, Vec<_>) = executing_thread_cpu_us.into_iter().unzip();
+        let (executing_thread_cpu_us, executing_thread_wall_time_us): (Vec<_>, Vec<_>) = executing_thread_duration_pairs.into_iter().unzip();
 
         let h = self.scheduler_thread_handle.take().unwrap();
         let scheduler_thread_duration_pairs = h.join().unwrap()?;

From 49e0bbf5185b0831fcedb3e9f699b28bfc0418da Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 13 Sep 2022 21:59:55 +0900
Subject: [PATCH 2528/3199] save

---
 runtime/src/bank.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index a2b144a7530a28..d66b01a6fe1390 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1446,8 +1446,8 @@ impl Scheduler {
         let scheduler_thread_duration_pairs = h.join().unwrap()?;
         let (scheduler_thread_cpu_us, scheduler_thread_wall_time_us) = (scheduler_thread_duration_pairs.0.as_micros(), scheduler_thread_duration_pairs.0.as_micros());
         let h = self.error_collector_thread_handle.take().unwrap();
-        let error_collector_thread_duration_pairs = h.join().unwrap()?
-            .as_micros();
+        let error_collector_thread_duration_pairs = h.join().unwrap()?;
+        let (error_collector_thread_cpu_us, error_collector_thread_wall_time_us) = (error_collector_thread_duration_pairs.0.as_micros(), error_collector_thread_duration_pairs.1.as_micros());
 
         info!("Scheduler::gracefully_stop(): slot: {} id_{:016x} cpu times: scheduler: {}us, error_collector: {}us, lanes: {}us = {:?}", self.slot.map(|s| format!("{}", s)).unwrap_or("-".into()), self.random_id, scheduler_thread_cpu_us, error_collector_thread_cpu_us, executing_thread_cpu_us.iter().sum::<u128>(), &executing_thread_cpu_us);
 

From 650ff4fb24233234138b08f6b668715c357a903f Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 13 Sep 2022 22:01:49 +0900
Subject: [PATCH 2529/3199] save

---
 runtime/src/bank.rs | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index d66b01a6fe1390..f97142a30af13c 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1356,7 +1356,7 @@ impl Default for Scheduler {
         let collected_errors_in_collector_thread = Arc::clone(&collected_errors);
 
         let error_collector_thread_handle = std::thread::Builder::new().name(format!("solScErrCol{:02}", 0)).spawn(move || {
-            let started = cpu_time::ThreadTime::now();
+            let started = (std::time::Instant::now(), cpu_time::ThreadTime::now());
             if max_thread_priority {
                 thread_priority::set_current_thread_priority(thread_priority::ThreadPriority::Max).unwrap();
             }
@@ -1372,13 +1372,13 @@ impl Default for Scheduler {
 
                 drop(ee);
             }
-            Ok(started.elapsed())
+            Ok((started.0.elapsed(), started.1.elapsed()))
         }).unwrap();
 
         let random_id = rand::thread_rng().gen::<u64>();
 
         let scheduler_thread_handle = std::thread::Builder::new().name("solScheduler".to_string()).spawn(move || {
-            let started = cpu_time::ThreadTime::now();
+            let started = (std::time::Instant::now(), cpu_time::ThreadTime::now());
             if max_thread_priority {
                 thread_priority::set_current_thread_priority(thread_priority::ThreadPriority::Max).unwrap();
             }
@@ -1403,7 +1403,7 @@ impl Default for Scheduler {
             drop(scheduled_ee_sender);
             drop(processed_ee_receiver);
 
-            Ok(started.elapsed())
+            Ok((started.0.elapsed(), started.1.elapsed()))
         }).unwrap();
 
         let s = Self {
@@ -1440,14 +1440,14 @@ impl Scheduler {
         }).collect();
         let mut executing_thread_duration_pairs = executing_thread_duration_pairs?;
         executing_thread_duration_pairs.sort();
-        let (executing_thread_cpu_us, executing_thread_wall_time_us): (Vec<_>, Vec<_>) = executing_thread_duration_pairs.into_iter().unzip();
+        let (executing_thread_wall_time_us, executing_thread_cpu_us): (Vec<_>, Vec<_>) = executing_thread_duration_pairs.into_iter().unzip();
 
         let h = self.scheduler_thread_handle.take().unwrap();
         let scheduler_thread_duration_pairs = h.join().unwrap()?;
-        let (scheduler_thread_cpu_us, scheduler_thread_wall_time_us) = (scheduler_thread_duration_pairs.0.as_micros(), scheduler_thread_duration_pairs.0.as_micros());
+        let (scheduler_thread_wall_time_us, scheduler_thread_cpu_us) = (scheduler_thread_duration_pairs.0.as_micros(), scheduler_thread_duration_pairs.0.as_micros());
         let h = self.error_collector_thread_handle.take().unwrap();
         let error_collector_thread_duration_pairs = h.join().unwrap()?;
-        let (error_collector_thread_cpu_us, error_collector_thread_wall_time_us) = (error_collector_thread_duration_pairs.0.as_micros(), error_collector_thread_duration_pairs.1.as_micros());
+        let (error_collector_thread_wall_time_us, error_collector_thread_cpu_us) = (error_collector_thread_duration_pairs.0.as_micros(), error_collector_thread_duration_pairs.1.as_micros());
 
         info!("Scheduler::gracefully_stop(): slot: {} id_{:016x} cpu times: scheduler: {}us, error_collector: {}us, lanes: {}us = {:?}", self.slot.map(|s| format!("{}", s)).unwrap_or("-".into()), self.random_id, scheduler_thread_cpu_us, error_collector_thread_cpu_us, executing_thread_cpu_us.iter().sum::<u128>(), &executing_thread_cpu_us);
 

From 3c9500a160536967a92781197b9009f4746e32c6 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 13 Sep 2022 22:02:09 +0900
Subject: [PATCH 2530/3199] save

---
 runtime/src/bank.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index f97142a30af13c..1591623c44106c 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1199,9 +1199,9 @@ impl AbiExample for BuiltinPrograms {
 #[derive(Debug)]
 struct Scheduler {
     random_id: u64,
-    scheduler_thread_handle: Option<std::thread::JoinHandle<Result<Duration>>>,
+    scheduler_thread_handle: Option<std::thread::JoinHandle<Result<(Duration Duration)>>>,
     executing_thread_handles: Option<Vec<std::thread::JoinHandle<Result<(Duration, Duration)>>>>,
-    error_collector_thread_handle: Option<std::thread::JoinHandle<Result<Duration>>>,
+    error_collector_thread_handle: Option<std::thread::JoinHandle<Result<(Duration, Duration)>>>,
     transaction_sender: Option<crossbeam_channel::Sender<solana_scheduler::SchedulablePayload>>,
     preloader: Arc<solana_scheduler::Preloader>,
     graceful_stop_initiated: bool,

From 5fb1ef533a4e0408e34863dedc179f5dde04b9e6 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 13 Sep 2022 22:02:19 +0900
Subject: [PATCH 2531/3199] save

---
 runtime/src/bank.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 1591623c44106c..82d24b784ba41b 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1199,7 +1199,7 @@ impl AbiExample for BuiltinPrograms {
 #[derive(Debug)]
 struct Scheduler {
     random_id: u64,
-    scheduler_thread_handle: Option<std::thread::JoinHandle<Result<(Duration Duration)>>>,
+    scheduler_thread_handle: Option<std::thread::JoinHandle<Result<(Duration, Duration)>>>,
     executing_thread_handles: Option<Vec<std::thread::JoinHandle<Result<(Duration, Duration)>>>>,
     error_collector_thread_handle: Option<std::thread::JoinHandle<Result<(Duration, Duration)>>>,
     transaction_sender: Option<crossbeam_channel::Sender<solana_scheduler::SchedulablePayload>>,

From 85fac2fb39c58b011128b711daefbdb660ecd2e6 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 13 Sep 2022 22:03:17 +0900
Subject: [PATCH 2532/3199] save

---
 runtime/src/bank.rs | 1 +
 1 file changed, 1 insertion(+)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 82d24b784ba41b..3a66ddd1aa37f9 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1449,6 +1449,7 @@ impl Scheduler {
         let error_collector_thread_duration_pairs = h.join().unwrap()?;
         let (error_collector_thread_wall_time_us, error_collector_thread_cpu_us) = (error_collector_thread_duration_pairs.0.as_micros(), error_collector_thread_duration_pairs.1.as_micros());
 
+        info!("Scheduler::gracefully_stop(): slot: {} id_{:016x} wall times: scheduler: {}us, error_collector: {}us, lanes: {}us = {:?}", self.slot.map(|s| format!("{}", s)).unwrap_or("-".into()), self.random_id, scheduler_thread_wall_time_us, error_collector_thread_wall_time_us, executing_thread_wall_time_us.iter().sum::<u128>(), &executing_thread_wall_time_us);
         info!("Scheduler::gracefully_stop(): slot: {} id_{:016x} cpu times: scheduler: {}us, error_collector: {}us, lanes: {}us = {:?}", self.slot.map(|s| format!("{}", s)).unwrap_or("-".into()), self.random_id, scheduler_thread_cpu_us, error_collector_thread_cpu_us, executing_thread_cpu_us.iter().sum::<u128>(), &executing_thread_cpu_us);
 
         Ok(())

From 9c3c496e7fb6c18065dd5b015d1868f78ab40be5 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Wed, 14 Sep 2022 12:40:42 +0900
Subject: [PATCH 2533/3199] save

---
 programs/bpf_loader/src/lib.rs |  2 ++
 runtime/src/bank.rs            | 14 +++++++-------
 2 files changed, 9 insertions(+), 7 deletions(-)

diff --git a/programs/bpf_loader/src/lib.rs b/programs/bpf_loader/src/lib.rs
index a01ee3ad6a1315..6cf4aad372825b 100644
--- a/programs/bpf_loader/src/lib.rs
+++ b/programs/bpf_loader/src/lib.rs
@@ -308,6 +308,7 @@ pub fn process_instruction(
     first_instruction_account: usize,
     invoke_context: &mut InvokeContext,
 ) -> Result<(), InstructionError> {
+    unreachable!();
     process_instruction_common(first_instruction_account, invoke_context, false)
 }
 
@@ -323,6 +324,7 @@ fn process_instruction_common(
     invoke_context: &mut InvokeContext,
     use_jit: bool,
 ) -> Result<(), InstructionError> {
+    assert!(use_jit);
     let log_collector = invoke_context.get_log_collector();
     let transaction_context = &invoke_context.transaction_context;
     let instruction_context = transaction_context.get_current_instruction_context()?;
diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 3a66ddd1aa37f9..52726acbb304d9 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1261,7 +1261,7 @@ impl Default for Scheduler {
             let bank = bank.clone();
 
             std::thread::Builder::new().name(format!("solScExLane{:02}", thx)).spawn(move || {
-            let started = (std::time::Instant::now(), cpu_time::ThreadTime::now());
+            let started = (cpu_time::ThreadTime::now(), std::time::Instant::now());
             if max_thread_priority {
                 thread_priority::set_current_thread_priority(thread_priority::ThreadPriority::Max).unwrap();
             }
@@ -1356,7 +1356,7 @@ impl Default for Scheduler {
         let collected_errors_in_collector_thread = Arc::clone(&collected_errors);
 
         let error_collector_thread_handle = std::thread::Builder::new().name(format!("solScErrCol{:02}", 0)).spawn(move || {
-            let started = (std::time::Instant::now(), cpu_time::ThreadTime::now());
+            let started = (cpu_time::ThreadTime::now(), std::time::Instant::now());
             if max_thread_priority {
                 thread_priority::set_current_thread_priority(thread_priority::ThreadPriority::Max).unwrap();
             }
@@ -1378,7 +1378,7 @@ impl Default for Scheduler {
         let random_id = rand::thread_rng().gen::<u64>();
 
         let scheduler_thread_handle = std::thread::Builder::new().name("solScheduler".to_string()).spawn(move || {
-            let started = (std::time::Instant::now(), cpu_time::ThreadTime::now());
+            let started = (cpu_time::ThreadTime::now(), std::time::Instant::now());
             if max_thread_priority {
                 thread_priority::set_current_thread_priority(thread_priority::ThreadPriority::Max).unwrap();
             }
@@ -1440,17 +1440,17 @@ impl Scheduler {
         }).collect();
         let mut executing_thread_duration_pairs = executing_thread_duration_pairs?;
         executing_thread_duration_pairs.sort();
-        let (executing_thread_wall_time_us, executing_thread_cpu_us): (Vec<_>, Vec<_>) = executing_thread_duration_pairs.into_iter().unzip();
+        let (executing_thread_cpu_us, executing_thread_wall_time_us): (Vec<_>, Vec<_>) = executing_thread_duration_pairs.into_iter().unzip();
 
         let h = self.scheduler_thread_handle.take().unwrap();
         let scheduler_thread_duration_pairs = h.join().unwrap()?;
-        let (scheduler_thread_wall_time_us, scheduler_thread_cpu_us) = (scheduler_thread_duration_pairs.0.as_micros(), scheduler_thread_duration_pairs.0.as_micros());
+        let (scheduler_thread_cpu_us, scheduler_wall_time_us) = (scheduler_thread_duration_pairs.0.as_micros(), scheduler_thread_duration_pairs.0.as_micros());
         let h = self.error_collector_thread_handle.take().unwrap();
         let error_collector_thread_duration_pairs = h.join().unwrap()?;
-        let (error_collector_thread_wall_time_us, error_collector_thread_cpu_us) = (error_collector_thread_duration_pairs.0.as_micros(), error_collector_thread_duration_pairs.1.as_micros());
+        let (error_collector_thread_cpu_us, error_collector_thread_wall_time_us) = (error_collector_thread_duration_pairs.0.as_micros(), error_collector_thread_duration_pairs.1.as_micros());
 
-        info!("Scheduler::gracefully_stop(): slot: {} id_{:016x} wall times: scheduler: {}us, error_collector: {}us, lanes: {}us = {:?}", self.slot.map(|s| format!("{}", s)).unwrap_or("-".into()), self.random_id, scheduler_thread_wall_time_us, error_collector_thread_wall_time_us, executing_thread_wall_time_us.iter().sum::<u128>(), &executing_thread_wall_time_us);
         info!("Scheduler::gracefully_stop(): slot: {} id_{:016x} cpu times: scheduler: {}us, error_collector: {}us, lanes: {}us = {:?}", self.slot.map(|s| format!("{}", s)).unwrap_or("-".into()), self.random_id, scheduler_thread_cpu_us, error_collector_thread_cpu_us, executing_thread_cpu_us.iter().sum::<u128>(), &executing_thread_cpu_us);
+        info!("Scheduler::gracefully_stop(): slot: {} id_{:016x} wall times: scheduler: {}us, error_collector: {}us, lanes: {}us = {:?}", self.slot.map(|s| format!("{}", s)).unwrap_or("-".into()), self.random_id, scheduler_thread_wall_time_us, error_collector_thread_wall_time_us, executing_thread_wall_time_us.iter().sum::<u128>(), &executing_thread_wall_time_us);
 
         Ok(())
     }

From 2da5d5099f3d8e1356cbb5b1326bad019b8c4b3c Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Wed, 14 Sep 2022 12:41:28 +0900
Subject: [PATCH 2534/3199] save

---
 runtime/src/bank.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 52726acbb304d9..db31dc0a4eb8e8 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1444,7 +1444,7 @@ impl Scheduler {
 
         let h = self.scheduler_thread_handle.take().unwrap();
         let scheduler_thread_duration_pairs = h.join().unwrap()?;
-        let (scheduler_thread_cpu_us, scheduler_wall_time_us) = (scheduler_thread_duration_pairs.0.as_micros(), scheduler_thread_duration_pairs.0.as_micros());
+        let (scheduler_thread_cpu_us, scheduler_thread_wall_time_us) = (scheduler_thread_duration_pairs.0.as_micros(), scheduler_thread_duration_pairs.0.as_micros());
         let h = self.error_collector_thread_handle.take().unwrap();
         let error_collector_thread_duration_pairs = h.join().unwrap()?;
         let (error_collector_thread_cpu_us, error_collector_thread_wall_time_us) = (error_collector_thread_duration_pairs.0.as_micros(), error_collector_thread_duration_pairs.1.as_micros());

From 7ba3ac5f79fa1d7717329ffdeae1a883f880a8fd Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Wed, 14 Sep 2022 13:00:25 +0900
Subject: [PATCH 2535/3199] save

---
 runtime/src/message_processor.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/src/message_processor.rs b/runtime/src/message_processor.rs
index e37265a1a08cbf..14fda0174e93d4 100644
--- a/runtime/src/message_processor.rs
+++ b/runtime/src/message_processor.rs
@@ -135,7 +135,7 @@ impl MessageProcessor {
             } else {
                 let mut time = Measure::start("execute_instruction");
                 let mut compute_units_consumed = 0;
-                let result = invoke_context.process_instruction(
+                let result = invoke_context.process_instruction_jit(
                     &instruction.data,
                     &instruction_accounts,
                     program_indices,

From dac71ad7eaaa4404f0f3433dd116cc104f10e84f Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Wed, 14 Sep 2022 13:02:48 +0900
Subject: [PATCH 2536/3199] save

---
 programs/bpf_loader/src/lib.rs   | 2 ++
 runtime/src/message_processor.rs | 2 +-
 2 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/programs/bpf_loader/src/lib.rs b/programs/bpf_loader/src/lib.rs
index 6cf4aad372825b..7beb0d137e58e8 100644
--- a/programs/bpf_loader/src/lib.rs
+++ b/programs/bpf_loader/src/lib.rs
@@ -304,6 +304,7 @@ pub fn create_vm<'a, 'b>(
     Ok(vm)
 }
 
+/*
 pub fn process_instruction(
     first_instruction_account: usize,
     invoke_context: &mut InvokeContext,
@@ -311,6 +312,7 @@ pub fn process_instruction(
     unreachable!();
     process_instruction_common(first_instruction_account, invoke_context, false)
 }
+*/
 
 pub fn process_instruction_jit(
     first_instruction_account: usize,
diff --git a/runtime/src/message_processor.rs b/runtime/src/message_processor.rs
index 14fda0174e93d4..e37265a1a08cbf 100644
--- a/runtime/src/message_processor.rs
+++ b/runtime/src/message_processor.rs
@@ -135,7 +135,7 @@ impl MessageProcessor {
             } else {
                 let mut time = Measure::start("execute_instruction");
                 let mut compute_units_consumed = 0;
-                let result = invoke_context.process_instruction_jit(
+                let result = invoke_context.process_instruction(
                     &instruction.data,
                     &instruction_accounts,
                     program_indices,

From 3c7548a7cc805bd5f5a2193e0ef716a22d7d9d81 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Wed, 14 Sep 2022 13:03:31 +0900
Subject: [PATCH 2537/3199] save

---
 programs/bpf_loader/src/upgradeable.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/programs/bpf_loader/src/upgradeable.rs b/programs/bpf_loader/src/upgradeable.rs
index 89aaed9fb5e260..32dbe162d0229c 100644
--- a/programs/bpf_loader/src/upgradeable.rs
+++ b/programs/bpf_loader/src/upgradeable.rs
@@ -1,6 +1,6 @@
 solana_sdk::declare_builtin!(
     solana_sdk::bpf_loader_upgradeable::ID,
     solana_bpf_loader_upgradeable_program,
-    solana_bpf_loader_program::process_instruction,
+    solana_bpf_loader_program::process_instruction_jit,
     upgradeable::id
 );

From ddc88c7ffe8c37285df6b91ace35050b8016cf6f Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Wed, 14 Sep 2022 13:04:05 +0900
Subject: [PATCH 2538/3199] save

---
 programs/bpf_loader/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/programs/bpf_loader/src/lib.rs b/programs/bpf_loader/src/lib.rs
index 7beb0d137e58e8..0395a3aeef607c 100644
--- a/programs/bpf_loader/src/lib.rs
+++ b/programs/bpf_loader/src/lib.rs
@@ -63,7 +63,7 @@ use {
 solana_sdk::declare_builtin!(
     solana_sdk::bpf_loader::ID,
     solana_bpf_loader_program,
-    solana_bpf_loader_program::process_instruction
+    solana_bpf_loader_program::process_instruction_jit
 );
 
 /// Errors returned by functions the BPF Loader registers with the VM

From 54c0d4efada460fb81ec29abf20877186163a559 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Wed, 14 Sep 2022 13:04:27 +0900
Subject: [PATCH 2539/3199] save

---
 programs/bpf_loader/src/deprecated.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/programs/bpf_loader/src/deprecated.rs b/programs/bpf_loader/src/deprecated.rs
index 3c1ee26602dda5..893e6db6dcfb66 100644
--- a/programs/bpf_loader/src/deprecated.rs
+++ b/programs/bpf_loader/src/deprecated.rs
@@ -1,6 +1,6 @@
 solana_sdk::declare_builtin!(
     solana_sdk::bpf_loader_deprecated::ID,
     solana_bpf_loader_deprecated_program,
-    solana_bpf_loader_program::process_instruction,
+    solana_bpf_loader_program::process_instruction_jit,
     deprecated::id
 );

From f10c7f75394789bb03fdb7dbf98f58646730c94b Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Wed, 14 Sep 2022 13:16:41 +0900
Subject: [PATCH 2540/3199] save

---
 runtime/src/bank.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index db31dc0a4eb8e8..07e78a5d05e6de 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1449,8 +1449,8 @@ impl Scheduler {
         let error_collector_thread_duration_pairs = h.join().unwrap()?;
         let (error_collector_thread_cpu_us, error_collector_thread_wall_time_us) = (error_collector_thread_duration_pairs.0.as_micros(), error_collector_thread_duration_pairs.1.as_micros());
 
-        info!("Scheduler::gracefully_stop(): slot: {} id_{:016x} cpu times: scheduler: {}us, error_collector: {}us, lanes: {}us = {:?}", self.slot.map(|s| format!("{}", s)).unwrap_or("-".into()), self.random_id, scheduler_thread_cpu_us, error_collector_thread_cpu_us, executing_thread_cpu_us.iter().sum::<u128>(), &executing_thread_cpu_us);
-        info!("Scheduler::gracefully_stop(): slot: {} id_{:016x} wall times: scheduler: {}us, error_collector: {}us, lanes: {}us = {:?}", self.slot.map(|s| format!("{}", s)).unwrap_or("-".into()), self.random_id, scheduler_thread_wall_time_us, error_collector_thread_wall_time_us, executing_thread_wall_time_us.iter().sum::<u128>(), &executing_thread_wall_time_us);
+        info!("Scheduler::gracefully_stop(): slot: {} id_{:016x} durations 1/2 (cpu ): scheduler: {}us, error_collector: {}us, lanes: {}us = {:?}", self.slot.map(|s| format!("{}", s)).unwrap_or("-".into()), self.random_id, scheduler_thread_cpu_us, error_collector_thread_cpu_us, executing_thread_cpu_us.iter().sum::<u128>(), &executing_thread_cpu_us);
+        info!("Scheduler::gracefully_stop(): slot: {} id_{:016x} durations 2/2 (wall): scheduler: {}us, error_collector: {}us, lanes: {}us = {:?}", self.slot.map(|s| format!("{}", s)).unwrap_or("-".into()), self.random_id, scheduler_thread_wall_time_us, error_collector_thread_wall_time_us, executing_thread_wall_time_us.iter().sum::<u128>(), &executing_thread_wall_time_us);
 
         Ok(())
     }

From 7aead72334ef99010b2132907bc6e7b422d47b0a Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Wed, 14 Sep 2022 13:26:12 +0900
Subject: [PATCH 2541/3199] save

---
 scheduler/src/lib.rs | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 0f6abaa3e948a3..bfd26b30755079 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1548,8 +1548,10 @@ impl ScheduleStage {
             let (mut from_len, mut from_exec_len) = (0, 0);
 
             loop {
-                let mut selection = TaskSelection::OnlyFromContended(if from_disconnected && runnable_queue.has_no_task() { usize::max_value() } else { usize::max_value() /*2*/ });
-                while selection.should_proceed() && executing_queue_count + provisioning_tracker_count < max_executing_queue_count {
+                let runnable_finished = from_disconnected && runnable_queue.has_no_task();
+
+                let mut selection = TaskSelection::OnlyFromContended(if runnable_finished { usize::max_value() } else { usize::max_value() /*2*/ });
+                while contended_count > 0 && selection.should_proceed() && executing_queue_count + provisioning_tracker_count < max_executing_queue_count {
                     let prefer_immediate = true; //provisioning_tracker_count / 4 > executing_queue_count;
 
                     let maybe_ee = Self::schedule_next_execution(
@@ -1593,7 +1595,7 @@ impl ScheduleStage {
                     }
                 }
                 let mut selection = TaskSelection::OnlyFromRunnable;
-                while selection.should_proceed() && executing_queue_count + provisioning_tracker_count < max_executing_queue_count {
+                while !runnable_finished && selection.should_proceed() && executing_queue_count + provisioning_tracker_count < max_executing_queue_count {
                     let prefer_immediate = true; //provisioning_tracker_count / 4 > executing_queue_count;
 
                     let maybe_ee = Self::schedule_next_execution(

From 5082ddea3108f4e1812a3b9b665d04721fe3f518 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Wed, 14 Sep 2022 13:42:25 +0900
Subject: [PATCH 2542/3199] save

---
 runtime/src/bank.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 07e78a5d05e6de..5dd9d4a6101090 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1444,7 +1444,7 @@ impl Scheduler {
 
         let h = self.scheduler_thread_handle.take().unwrap();
         let scheduler_thread_duration_pairs = h.join().unwrap()?;
-        let (scheduler_thread_cpu_us, scheduler_thread_wall_time_us) = (scheduler_thread_duration_pairs.0.as_micros(), scheduler_thread_duration_pairs.0.as_micros());
+        let (scheduler_thread_cpu_us, scheduler_thread_wall_time_us) = (scheduler_thread_duration_pairs.0.as_micros(), scheduler_thread_duration_pairs.1.as_micros());
         let h = self.error_collector_thread_handle.take().unwrap();
         let error_collector_thread_duration_pairs = h.join().unwrap()?;
         let (error_collector_thread_cpu_us, error_collector_thread_wall_time_us) = (error_collector_thread_duration_pairs.0.as_micros(), error_collector_thread_duration_pairs.1.as_micros());

From a40014f63869bed55ea44ca766f4a74d55f2fdd0 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Wed, 14 Sep 2022 14:21:01 +0900
Subject: [PATCH 2543/3199] save

---
 scheduler/src/lib.rs | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index bfd26b30755079..32673c1458ddce 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1499,6 +1499,8 @@ impl ScheduleStage {
                 .unwrap()
         }).collect::<Vec<_>>();
 
+        let no_aggresive_contended = std::env::var("NO_AGGRESSIVE_CONTENDED").is_ok();
+
         let (mut from_disconnected, mut from_exec_disconnected, mut no_more_work): (bool, bool, bool) = Default::default();
 
         loop {
@@ -1576,12 +1578,10 @@ impl ScheduleStage {
                         debug!("schedule_once id_{:016x} [C] ch(prev: {}, exec: {}|{}), r: {}, u/c: {}/{}, (imm+provi)/max: ({}+{})/{} s: {} l(s+f): {}+{}", random_id, (if from_disconnected { "-".to_string() } else { format!("{}", from_prev.len()) }), to_execute_substage.len(), from_exec.len(), runnable_queue.task_count(), address_book.uncontended_task_ids.len(), contended_count, executing_queue_count, provisioning_tracker_count, max_executing_queue_count, address_book.stuck_tasks.len(), processed_count, failed_lock_count);
                         break;
                     }
-                    /*
-                    if !from_exec.is_empty() {
+                    if no_aggresive_contended && !from_exec.is_empty() {
                         trace!("abort aggressive contended queue processing due to non-empty from_exec");
                         break;
                     }
-                    */
 
                     interval_count += 1;
                     if interval_count % 100 == 0 {

From 3c85e142bc32330b3fad4c2d365ef89b1883f2e5 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 15 Sep 2022 13:22:46 +0900
Subject: [PATCH 2544/3199] save

---
 runtime/src/bank.rs | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 5dd9d4a6101090..54fea15bf8d07a 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1196,6 +1196,11 @@ impl AbiExample for BuiltinPrograms {
     }
 }
 
+struct SchedulerPool {
+}
+
+const SCHEDULER_POOL = RwLock::new(SchedulerPool::new());
+
 #[derive(Debug)]
 struct Scheduler {
     random_id: u64,
@@ -6531,6 +6536,7 @@ impl Bank {
 
     /// Process a batch of transactions.
     #[must_use]
+    #[inline(never)]
     pub fn load_execute_and_commit_transactions(
         &self,
         batch: &TransactionBatch,

From 592f2af6f4a220344a05f8c6bff9a9882a689def Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 15 Sep 2022 13:23:37 +0900
Subject: [PATCH 2545/3199] save

---
 runtime/src/bank.rs | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 54fea15bf8d07a..61db883a39dd24 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1199,6 +1199,12 @@ impl AbiExample for BuiltinPrograms {
 struct SchedulerPool {
 }
 
+impl SchedulerPool {
+    fn new() -> Self {
+        Self {}
+    }
+}
+
 const SCHEDULER_POOL = RwLock::new(SchedulerPool::new());
 
 #[derive(Debug)]

From 0cb2377107b74b6f006157816dced6932f0ce8bd Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 15 Sep 2022 13:24:04 +0900
Subject: [PATCH 2546/3199] save

---
 runtime/src/bank.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 61db883a39dd24..aacc05f5194e3b 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1205,7 +1205,7 @@ impl SchedulerPool {
     }
 }
 
-const SCHEDULER_POOL = RwLock::new(SchedulerPool::new());
+const SCHEDULER_POOL: std::sync::RwLock<SchedulerPool> = RwLock::new(SchedulerPool::new());
 
 #[derive(Debug)]
 struct Scheduler {

From 6ec6a5b884dd2ecaa88ac06a19458f482a429a17 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 15 Sep 2022 13:24:23 +0900
Subject: [PATCH 2547/3199] save

---
 runtime/src/bank.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index aacc05f5194e3b..0f76920f080fd0 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1200,7 +1200,7 @@ struct SchedulerPool {
 }
 
 impl SchedulerPool {
-    fn new() -> Self {
+    const fn new() -> Self {
         Self {}
     }
 }

From dd8b01c9e7167335d66ca7a7fb757abb4c7f0100 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 15 Sep 2022 13:25:20 +0900
Subject: [PATCH 2548/3199] save

---
 runtime/src/bank.rs | 1 +
 1 file changed, 1 insertion(+)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 0f76920f080fd0..d3d001d96d0a2d 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1197,6 +1197,7 @@ impl AbiExample for BuiltinPrograms {
 }
 
 struct SchedulerPool {
+    schedulers: Vec<Scheduler>,
 }
 
 impl SchedulerPool {

From 387dd6c7a06f2cf867c387529e84221d9929959b Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 15 Sep 2022 13:25:52 +0900
Subject: [PATCH 2549/3199] save

---
 runtime/src/bank.rs | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index d3d001d96d0a2d..137870ad022578 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1202,7 +1202,9 @@ struct SchedulerPool {
 
 impl SchedulerPool {
     const fn new() -> Self {
-        Self {}
+        Self {
+            schedulers: Vec<Scheduler>,
+        }
     }
 }
 

From 9be23b52e28d83a619ee7a9bac9f5073e43ec5a6 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 15 Sep 2022 13:26:08 +0900
Subject: [PATCH 2550/3199] save

---
 runtime/src/bank.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 137870ad022578..ec10b3958a5445 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1203,7 +1203,7 @@ struct SchedulerPool {
 impl SchedulerPool {
     const fn new() -> Self {
         Self {
-            schedulers: Vec<Scheduler>,
+            schedulers: Vec::new(),
         }
     }
 }

From 5521c2b7dcdef52096acf66c5ad159dd1088327e Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 15 Sep 2022 13:26:39 +0900
Subject: [PATCH 2551/3199] save

---
 runtime/src/bank.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index ec10b3958a5445..99c4bbe16b069e 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1208,7 +1208,7 @@ impl SchedulerPool {
     }
 }
 
-const SCHEDULER_POOL: std::sync::RwLock<SchedulerPool> = RwLock::new(SchedulerPool::new());
+const SCHEDULER_POOL: RwLock<SchedulerPool> = RwLock::new(SchedulerPool::new());
 
 #[derive(Debug)]
 struct Scheduler {

From 254cfb353a56534feda3259919019691aeef91e1 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 15 Sep 2022 13:33:01 +0900
Subject: [PATCH 2552/3199] save

---
 runtime/src/bank.rs | 10 +++++++++-
 1 file changed, 9 insertions(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 99c4bbe16b069e..ddbdf1c0d1ee40 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1206,9 +1206,17 @@ impl SchedulerPool {
             schedulers: Vec::new(),
         }
     }
+
+    fn take_from_pool(&mut self) -> Scheduler {
+        todo!()
+    }
+
+    fn return_to_pool(&mut self, Scheduler) {
+        todo!()
+    }
 }
 
-const SCHEDULER_POOL: RwLock<SchedulerPool> = RwLock::new(SchedulerPool::new());
+const SCHEDULER_POOL: RwLock<SchedulerPool> = std::sync::Mutex::new(SchedulerPool::new());
 
 #[derive(Debug)]
 struct Scheduler {

From 5708e27297d15550e8c9272aefb5d7236e06356b Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 15 Sep 2022 13:33:36 +0900
Subject: [PATCH 2553/3199] save

---
 runtime/src/bank.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index ddbdf1c0d1ee40..ba7c0e92fb89f8 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1211,7 +1211,7 @@ impl SchedulerPool {
         todo!()
     }
 
-    fn return_to_pool(&mut self, Scheduler) {
+    fn return_to_pool(&mut self, _: Scheduler) {
         todo!()
     }
 }

From 53effe5575d61cd6916e3efc4f4a2fe997e81ec3 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 15 Sep 2022 13:33:50 +0900
Subject: [PATCH 2554/3199] save

---
 runtime/src/bank.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index ba7c0e92fb89f8..b0485be55d402b 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1216,7 +1216,7 @@ impl SchedulerPool {
     }
 }
 
-const SCHEDULER_POOL: RwLock<SchedulerPool> = std::sync::Mutex::new(SchedulerPool::new());
+const SCHEDULER_POOL: Mutex<SchedulerPool> = std::sync::Mutex::new(SchedulerPool::new());
 
 #[derive(Debug)]
 struct Scheduler {

From 11c52b3a10dd63fee4c7257f6b30baebb496d5e4 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 15 Sep 2022 13:34:01 +0900
Subject: [PATCH 2555/3199] save

---
 runtime/src/bank.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index b0485be55d402b..7af3eaa8afcb82 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1216,7 +1216,7 @@ impl SchedulerPool {
     }
 }
 
-const SCHEDULER_POOL: Mutex<SchedulerPool> = std::sync::Mutex::new(SchedulerPool::new());
+const SCHEDULER_POOL: std::sync::Mutex<SchedulerPool> = std::sync::Mutex::new(SchedulerPool::new());
 
 #[derive(Debug)]
 struct Scheduler {

From 5585cf13d16dba710491c0319975eb007cc8f6ab Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 15 Sep 2022 15:15:44 +0900
Subject: [PATCH 2556/3199] save

---
 runtime/src/bank.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 7af3eaa8afcb82..e04eb3bc910803 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1197,7 +1197,7 @@ impl AbiExample for BuiltinPrograms {
 }
 
 struct SchedulerPool {
-    schedulers: Vec<Scheduler>,
+    schedulers: Vec<Arc<Scheduler>>,
 }
 
 impl SchedulerPool {

From 250cee35ecec8a1301d63f92ef29496affa2197a Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 15 Sep 2022 15:17:01 +0900
Subject: [PATCH 2557/3199] save

---
 runtime/src/bank.rs | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index e04eb3bc910803..70fa9d1c0e6b5c 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1207,11 +1207,12 @@ impl SchedulerPool {
         }
     }
 
-    fn take_from_pool(&mut self) -> Scheduler {
+    fn take_from_pool(&mut self) -> Arc<Scheduler> {
         todo!()
     }
 
-    fn return_to_pool(&mut self, _: Scheduler) {
+    fn return_to_pool(&mut self, scheduler: Arc<Scheduler>) {
+        assert_eq!(1, Arc::strong_count(&scheduler));
         todo!()
     }
 }

From fb6173f6b0cab7b1a0e55aad2e3d26e2f469a891 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 15 Sep 2022 15:18:50 +0900
Subject: [PATCH 2558/3199] save

---
 runtime/src/bank.rs | 11 ++++++++++-
 1 file changed, 10 insertions(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 70fa9d1c0e6b5c..12e5fe46c428e9 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1207,8 +1207,17 @@ impl SchedulerPool {
         }
     }
 
+    fn create(&mut self) {
+        todo!();
+    }
+
     fn take_from_pool(&mut self) -> Arc<Scheduler> {
-        todo!()
+        if let Some(scheduler) = self.schedulers.pop() {
+            scheduler
+        } else {
+            self.create();
+            self.take_from_pool()
+        }
     }
 
     fn return_to_pool(&mut self, scheduler: Arc<Scheduler>) {

From e69ffe5fc915f3cba35eaf47e1b8e692955c11a3 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 15 Sep 2022 15:19:50 +0900
Subject: [PATCH 2559/3199] save

---
 runtime/src/bank.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 12e5fe46c428e9..e81e1b23c9d156 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1222,7 +1222,7 @@ impl SchedulerPool {
 
     fn return_to_pool(&mut self, scheduler: Arc<Scheduler>) {
         assert_eq!(1, Arc::strong_count(&scheduler));
-        todo!()
+        self.schedulers.push(scheduler);
     }
 }
 

From f230508e7918439c5cd6aec13d03e49b970c22e1 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 15 Sep 2022 15:20:45 +0900
Subject: [PATCH 2560/3199] save

---
 runtime/src/bank.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index e81e1b23c9d156..47023a4e5c9f4d 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1208,7 +1208,7 @@ impl SchedulerPool {
     }
 
     fn create(&mut self) {
-        todo!();
+        self.schedulers.push(Default::default());
     }
 
     fn take_from_pool(&mut self) -> Arc<Scheduler> {

From 5eb72cdfa9fe40131f3fd780e6e9190bd90fb7fa Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 15 Sep 2022 15:27:36 +0900
Subject: [PATCH 2561/3199] save

---
 scheduler/src/lib.rs | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 32673c1458ddce..51049804ecc9b2 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1730,3 +1730,8 @@ pub struct SchedulablePayload(pub TaskInQueue);
 pub struct ExecutablePayload(pub Box<ExecutionEnvironment>);
 pub struct UnlockablePayload(pub Box<ExecutionEnvironment>);
 pub struct ExaminablePayload(pub Box<ExecutionEnvironment>);
+
+enum Flushable {
+    Payload<T>,
+    Flush,
+};

From 593b254f4cc948fe2925ce55d684fb45728929f4 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 15 Sep 2022 15:27:54 +0900
Subject: [PATCH 2562/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 51049804ecc9b2..56eb27ac2f0ee3 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1732,6 +1732,6 @@ pub struct UnlockablePayload(pub Box<ExecutionEnvironment>);
 pub struct ExaminablePayload(pub Box<ExecutionEnvironment>);
 
 enum Flushable {
-    Payload<T>,
+    Payload(T),
     Flush,
 };

From a41b451dc14304bf2af32e6bbd53a1065e6d33aa Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 15 Sep 2022 15:28:02 +0900
Subject: [PATCH 2563/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 56eb27ac2f0ee3..6eff5cf3bf2eb5 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1731,7 +1731,7 @@ pub struct ExecutablePayload(pub Box<ExecutionEnvironment>);
 pub struct UnlockablePayload(pub Box<ExecutionEnvironment>);
 pub struct ExaminablePayload(pub Box<ExecutionEnvironment>);
 
-enum Flushable {
+enum Flushable<T> {
     Payload(T),
     Flush,
 };

From d9899a808a0afcf87089874acf8129d625456428 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 15 Sep 2022 15:28:08 +0900
Subject: [PATCH 2564/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 6eff5cf3bf2eb5..aa493617f37930 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1734,4 +1734,4 @@ pub struct ExaminablePayload(pub Box<ExecutionEnvironment>);
 enum Flushable<T> {
     Payload(T),
     Flush,
-};
+}

From e45e8f988b8a7d172a412e085c36d5b4e64c2692 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 15 Sep 2022 15:28:33 +0900
Subject: [PATCH 2565/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index aa493617f37930..a1fcae26bcad3c 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1729,7 +1729,7 @@ impl ScheduleStage {
 pub struct SchedulablePayload(pub TaskInQueue);
 pub struct ExecutablePayload(pub Box<ExecutionEnvironment>);
 pub struct UnlockablePayload(pub Box<ExecutionEnvironment>);
-pub struct ExaminablePayload(pub Box<ExecutionEnvironment>);
+pub struct ExaminablePayload(pub Flushable<Box<ExecutionEnvironment>>);
 
 enum Flushable<T> {
     Payload(T),

From 13b5d6126b5d0f8d9dda4078686856d81f4ea37a Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 15 Sep 2022 15:29:46 +0900
Subject: [PATCH 2566/3199] save

---
 scheduler/src/lib.rs | 19 +++++++++++++------
 1 file changed, 13 insertions(+), 6 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index a1fcae26bcad3c..6b21c2b9060e03 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1457,12 +1457,19 @@ impl ScheduleStage {
                     unsafe impl NotAtScheduleThread for NotAtTopOfScheduleThread {}
                     let nast = NotAtTopOfScheduleThread;
 
-                    while let Ok(ExaminablePayload(a)) = ee_receiver.recv() {
-                        assert!(a.task.lock_attempts_not_mut(nast).is_empty());
-                        //assert!(a.task.sequence_time() != usize::max_value());
-                        //let lock_attempts = std::mem::take(&mut a.lock_attempts);
-                        //drop(lock_attempts);
-                        //TaskInQueue::get_mut(&mut a.task).unwrap();
+                    while let Ok(flushable_payload) = ee_receiver.recv() {
+                        match flushable_payload {
+                            Flush => {
+                                continue;
+                            }
+                            Payload(ExaminablePayload(a)) => {
+                                assert!(a.task.lock_attempts_not_mut(nast).is_empty());
+                                //assert!(a.task.sequence_time() != usize::max_value());
+                                //let lock_attempts = std::mem::take(&mut a.lock_attempts);
+                                //drop(lock_attempts);
+                                //TaskInQueue::get_mut(&mut a.task).unwrap();
+                            }
+                        }
                     }
                     assert_eq!(ee_receiver.len(), 0);
                     Ok::<(), ()>(())

From 2d46ce24ace2e532efb8c5cd824557794e9da1a4 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 15 Sep 2022 15:30:00 +0900
Subject: [PATCH 2567/3199] save

---
 scheduler/src/lib.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 6b21c2b9060e03..1136fe69037e3a 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1459,10 +1459,10 @@ impl ScheduleStage {
 
                     while let Ok(flushable_payload) = ee_receiver.recv() {
                         match flushable_payload {
-                            Flush => {
+                            Flashable::Flush => {
                                 continue;
                             }
-                            Payload(ExaminablePayload(a)) => {
+                            Flashable::Payload(ExaminablePayload(a)) => {
                                 assert!(a.task.lock_attempts_not_mut(nast).is_empty());
                                 //assert!(a.task.sequence_time() != usize::max_value());
                                 //let lock_attempts = std::mem::take(&mut a.lock_attempts);

From a2ad970e37525d50f71efd208842d18e2ba194c0 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 15 Sep 2022 15:30:35 +0900
Subject: [PATCH 2568/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 1136fe69037e3a..19e13ebf0bf590 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1736,7 +1736,7 @@ impl ScheduleStage {
 pub struct SchedulablePayload(pub TaskInQueue);
 pub struct ExecutablePayload(pub Box<ExecutionEnvironment>);
 pub struct UnlockablePayload(pub Box<ExecutionEnvironment>);
-pub struct ExaminablePayload(pub Flushable<Box<ExecutionEnvironment>>);
+pub struct Flushable<ExaminablePayload(pub Box<ExecutionEnvironment>)>;
 
 enum Flushable<T> {
     Payload(T),

From 873ad2bd984fa1cc6660b73d30d3f22c0cb8c099 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 15 Sep 2022 15:31:20 +0900
Subject: [PATCH 2569/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 19e13ebf0bf590..720891ca83158c 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1736,7 +1736,7 @@ impl ScheduleStage {
 pub struct SchedulablePayload(pub TaskInQueue);
 pub struct ExecutablePayload(pub Box<ExecutionEnvironment>);
 pub struct UnlockablePayload(pub Box<ExecutionEnvironment>);
-pub struct Flushable<ExaminablePayload(pub Box<ExecutionEnvironment>)>;
+pub Flushable<struct ExaminablePayload(pub Box<ExecutionEnvironment>)>;
 
 enum Flushable<T> {
     Payload(T),

From 3f3b0dfe51777b8800bdde2065c72f669c360b79 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 15 Sep 2022 15:32:01 +0900
Subject: [PATCH 2570/3199] save

---
 scheduler/src/lib.rs | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 720891ca83158c..d549b565646f46 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1457,12 +1457,12 @@ impl ScheduleStage {
                     unsafe impl NotAtScheduleThread for NotAtTopOfScheduleThread {}
                     let nast = NotAtTopOfScheduleThread;
 
-                    while let Ok(flushable_payload) = ee_receiver.recv() {
-                        match flushable_payload {
+                    while let Ok(ExaminablePayload(a)) = ee_receiver.recv() {
+                        match a {
                             Flashable::Flush => {
                                 continue;
                             }
-                            Flashable::Payload(ExaminablePayload(a)) => {
+                            Flashable::Payload() => {
                                 assert!(a.task.lock_attempts_not_mut(nast).is_empty());
                                 //assert!(a.task.sequence_time() != usize::max_value());
                                 //let lock_attempts = std::mem::take(&mut a.lock_attempts);
@@ -1736,7 +1736,7 @@ impl ScheduleStage {
 pub struct SchedulablePayload(pub TaskInQueue);
 pub struct ExecutablePayload(pub Box<ExecutionEnvironment>);
 pub struct UnlockablePayload(pub Box<ExecutionEnvironment>);
-pub Flushable<struct ExaminablePayload(pub Box<ExecutionEnvironment>)>;
+pub struct ExaminablePayload(pub Flushable<Box<ExecutionEnvironment>>);
 
 enum Flushable<T> {
     Payload(T),

From c95c902073669808f9ee169effe3a958e22329c0 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 15 Sep 2022 15:35:49 +0900
Subject: [PATCH 2571/3199] save

---
 scheduler/src/lib.rs | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index d549b565646f46..9db0e4ce56fbcb 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1459,16 +1459,16 @@ impl ScheduleStage {
 
                     while let Ok(ExaminablePayload(a)) = ee_receiver.recv() {
                         match a {
-                            Flashable::Flush => {
+                            Flush => {
                                 continue;
-                            }
-                            Flashable::Payload() => {
+                            },
+                            Payload(a) => {
                                 assert!(a.task.lock_attempts_not_mut(nast).is_empty());
                                 //assert!(a.task.sequence_time() != usize::max_value());
                                 //let lock_attempts = std::mem::take(&mut a.lock_attempts);
                                 //drop(lock_attempts);
                                 //TaskInQueue::get_mut(&mut a.task).unwrap();
-                            }
+                            },
                         }
                     }
                     assert_eq!(ee_receiver.len(), 0);

From 651f12ff18524cb8eac5e428470248f8325cd141 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 15 Sep 2022 15:36:02 +0900
Subject: [PATCH 2572/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 9db0e4ce56fbcb..73f3079812c66e 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1462,7 +1462,7 @@ impl ScheduleStage {
                             Flush => {
                                 continue;
                             },
-                            Payload(a) => {
+                            Flushable::Payload(a) => {
                                 assert!(a.task.lock_attempts_not_mut(nast).is_empty());
                                 //assert!(a.task.sequence_time() != usize::max_value());
                                 //let lock_attempts = std::mem::take(&mut a.lock_attempts);

From 25bf602aafc1229c540c79fbc9e4cf05e0914306 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 15 Sep 2022 15:36:48 +0900
Subject: [PATCH 2573/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 73f3079812c66e..e027195ed4e778 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1520,7 +1520,7 @@ impl ScheduleStage {
                            executing_queue_count = executing_queue_count.checked_sub(1).unwrap();
                            processed_count = processed_count.checked_add(1).unwrap();
                            Self::commit_processed_execution(ast, &mut processed_execution_environment, address_book, &mut commit_clock, &mut provisioning_tracker_count);
-                           to_next_stage.send(ExaminablePayload(processed_execution_environment)).unwrap();
+                           to_next_stage.send(ExaminablePayload(Flushable::Payload(processed_execution_environment))).unwrap();
                        } else {
                            assert_eq!(from_exec.len(), 0);
                            from_exec_disconnected = true;

From c41a92b3c38c6770a7fb4013243548450230d1bf Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 15 Sep 2022 15:37:07 +0900
Subject: [PATCH 2574/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index e027195ed4e778..a08c2a5ec8bd6c 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1675,7 +1675,7 @@ impl ScheduleStage {
                             &mut commit_clock,
                             &mut provisioning_tracker_count,
                         );
-                        to_next_stage.send(ExaminablePayload(processed_execution_environment)).unwrap();
+                        to_next_stage.send(ExaminablePayload(Flushable::Payload(processed_execution_environment))).unwrap();
                     }
                     if !empty_from {
                         let task = from_prev.recv().unwrap().0;

From d359d5f764086293d5869be432bacd6ac162b3b4 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 15 Sep 2022 15:37:18 +0900
Subject: [PATCH 2575/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index a08c2a5ec8bd6c..8631172e8e59cb 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1738,7 +1738,7 @@ pub struct ExecutablePayload(pub Box<ExecutionEnvironment>);
 pub struct UnlockablePayload(pub Box<ExecutionEnvironment>);
 pub struct ExaminablePayload(pub Flushable<Box<ExecutionEnvironment>>);
 
-enum Flushable<T> {
+pub enum Flushable<T> {
     Payload(T),
     Flush,
 }

From fc77d91e16549c05e0e6ba44a993aa480b1f8622 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 15 Sep 2022 16:16:47 +0900
Subject: [PATCH 2576/3199] save

---
 scheduler/src/lib.rs | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 8631172e8e59cb..c81b7a71c01853 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1688,6 +1688,8 @@ impl ScheduleStage {
             assert!(!select_skipped || executing_queue_count > 0);
         }
         drop(to_next_stage);
+        let pair = Arc::new((Mutex::new(false), Condvar::new()));
+        to_next_stage.send(ExecutablePayload(Flushable::Flush(pair))).unwrap();
         drop(ee_sender);
         drop(task_sender);
         drop(task_receiver);

From b87001f1cdf403e7132a30906b271988c4fff0f0 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 15 Sep 2022 16:17:13 +0900
Subject: [PATCH 2577/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index c81b7a71c01853..b74cb122d600bc 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1688,7 +1688,7 @@ impl ScheduleStage {
             assert!(!select_skipped || executing_queue_count > 0);
         }
         drop(to_next_stage);
-        let pair = Arc::new((Mutex::new(false), Condvar::new()));
+        let pair = std::sync::Arc::new((std::sync::Mutex::new(false), std::sync::Condvar::new()));
         to_next_stage.send(ExecutablePayload(Flushable::Flush(pair))).unwrap();
         drop(ee_sender);
         drop(task_sender);

From af20ea82dec715a9e30a5475a73fd37104e85f59 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 15 Sep 2022 16:17:33 +0900
Subject: [PATCH 2578/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index b74cb122d600bc..6132182e54a349 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1742,5 +1742,5 @@ pub struct ExaminablePayload(pub Flushable<Box<ExecutionEnvironment>>);
 
 pub enum Flushable<T> {
     Payload(T),
-    Flush,
+    Flush(usize),
 }

From 92b33b0ba51bdf179963c4b68052e8960c92e48f Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 15 Sep 2022 16:18:06 +0900
Subject: [PATCH 2579/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 6132182e54a349..60f67e2dd8d138 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1689,7 +1689,7 @@ impl ScheduleStage {
         }
         drop(to_next_stage);
         let pair = std::sync::Arc::new((std::sync::Mutex::new(false), std::sync::Condvar::new()));
-        to_next_stage.send(ExecutablePayload(Flushable::Flush(pair))).unwrap();
+        to_next_stage.send(ExaminablePayload(Flushable::Flush(pair))).unwrap();
         drop(ee_sender);
         drop(task_sender);
         drop(task_receiver);

From f61ab32a5c9f84ecba8bad8918c5ca24d4d4e1ac Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 15 Sep 2022 16:18:34 +0900
Subject: [PATCH 2580/3199] save

---
 scheduler/src/lib.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 60f67e2dd8d138..6570ff4fb046eb 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1687,7 +1687,7 @@ impl ScheduleStage {
             }
             assert!(!select_skipped || executing_queue_count > 0);
         }
-        drop(to_next_stage);
+        //drop(to_next_stage);
         let pair = std::sync::Arc::new((std::sync::Mutex::new(false), std::sync::Condvar::new()));
         to_next_stage.send(ExaminablePayload(Flushable::Flush(pair))).unwrap();
         drop(ee_sender);
@@ -1742,5 +1742,5 @@ pub struct ExaminablePayload(pub Flushable<Box<ExecutionEnvironment>>);
 
 pub enum Flushable<T> {
     Payload(T),
-    Flush(usize),
+    Flush(std::sync::Arc<(Mutex<bool>, Condvar)>),
 }

From dd9ddc976b3deff75263d5978f24a966b2caefe0 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 15 Sep 2022 16:18:53 +0900
Subject: [PATCH 2581/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 6570ff4fb046eb..e1d508d70b04e5 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1742,5 +1742,5 @@ pub struct ExaminablePayload(pub Flushable<Box<ExecutionEnvironment>>);
 
 pub enum Flushable<T> {
     Payload(T),
-    Flush(std::sync::Arc<(Mutex<bool>, Condvar)>),
+    Flush(std::sync::Arc<(std::sync::Mutex<bool>, std::sync::Condvar)>),
 }

From 65cc4cdeb2d049d5fdc4de53deb1101ca699fe69 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 15 Sep 2022 16:20:27 +0900
Subject: [PATCH 2582/3199] save

---
 scheduler/src/lib.rs | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index e1d508d70b04e5..82c2454b223919 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1459,7 +1459,12 @@ impl ScheduleStage {
 
                     while let Ok(ExaminablePayload(a)) = ee_receiver.recv() {
                         match a {
-                            Flush => {
+                            Flush((lock, cvar)) => {
+                                let started = lock.lock().unwrap();
+                                while !*started {
+                                    started = cvar.wait(started).unwrap();
+                                }
+
                                 continue;
                             },
                             Flushable::Payload(a) => {

From db398e30875771f056a61235adf04fdf4b81917b Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 15 Sep 2022 16:20:44 +0900
Subject: [PATCH 2583/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 82c2454b223919..b161c93e26aae9 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1459,7 +1459,7 @@ impl ScheduleStage {
 
                     while let Ok(ExaminablePayload(a)) = ee_receiver.recv() {
                         match a {
-                            Flush((lock, cvar)) => {
+                            Flushable::Flush((lock, cvar)) => {
                                 let started = lock.lock().unwrap();
                                 while !*started {
                                     started = cvar.wait(started).unwrap();

From a459a62796e88b250bd24c5555a3f2f1011101f1 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 15 Sep 2022 16:21:29 +0900
Subject: [PATCH 2584/3199] save

---
 scheduler/src/lib.rs | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index b161c93e26aae9..ef20e9754f87aa 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1459,7 +1459,8 @@ impl ScheduleStage {
 
                     while let Ok(ExaminablePayload(a)) = ee_receiver.recv() {
                         match a {
-                            Flushable::Flush((lock, cvar)) => {
+                            Flushable::Flush(pair) => {
+                                let (lock, cvar) = &*pair;
                                 let started = lock.lock().unwrap();
                                 while !*started {
                                     started = cvar.wait(started).unwrap();

From ac9b3f7ccf6c31b97708ce8fd63eb3d264e744cc Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 15 Sep 2022 16:21:39 +0900
Subject: [PATCH 2585/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index ef20e9754f87aa..de33028ff5ea7c 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1461,7 +1461,7 @@ impl ScheduleStage {
                         match a {
                             Flushable::Flush(pair) => {
                                 let (lock, cvar) = &*pair;
-                                let started = lock.lock().unwrap();
+                                let mut started = lock.lock().unwrap();
                                 while !*started {
                                     started = cvar.wait(started).unwrap();
                                 }

From 0df94b628def2be8a7d439b227dd7d070e8f78de Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 15 Sep 2022 16:27:56 +0900
Subject: [PATCH 2586/3199] save

---
 scheduler/src/lib.rs | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index de33028ff5ea7c..2c1b1b736471f5 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1461,10 +1461,7 @@ impl ScheduleStage {
                         match a {
                             Flushable::Flush(pair) => {
                                 let (lock, cvar) = &*pair;
-                                let mut started = lock.lock().unwrap();
-                                while !*started {
-                                    started = cvar.wait(started).unwrap();
-                                }
+                                cvar.wait_while(lock.lock().unwrap(), |remaining_count| remaining_count == 0).unwrap();
 
                                 continue;
                             },

From 98883408dd935772484accbfa74c95815000e90f Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 15 Sep 2022 16:28:32 +0900
Subject: [PATCH 2587/3199] save

---
 scheduler/src/lib.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 2c1b1b736471f5..1ad2dbd20dbbd8 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1461,7 +1461,7 @@ impl ScheduleStage {
                         match a {
                             Flushable::Flush(pair) => {
                                 let (lock, cvar) = &*pair;
-                                cvar.wait_while(lock.lock().unwrap(), |remaining_count| remaining_count == 0).unwrap();
+                                cvar.wait_while(lock.lock().unwrap(), |remaining_threads| remaining_threads == 0).unwrap();
 
                                 continue;
                             },
@@ -1745,5 +1745,5 @@ pub struct ExaminablePayload(pub Flushable<Box<ExecutionEnvironment>>);
 
 pub enum Flushable<T> {
     Payload(T),
-    Flush(std::sync::Arc<(std::sync::Mutex<bool>, std::sync::Condvar)>),
+    Flush(std::sync::Arc<(std::sync::Mutex<usize>, std::sync::Condvar)>),
 }

From 842a352b854b876134d9346c154ea066a8b26254 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 15 Sep 2022 16:29:05 +0900
Subject: [PATCH 2588/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 1ad2dbd20dbbd8..ea1ff70203c8d4 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1691,7 +1691,7 @@ impl ScheduleStage {
             assert!(!select_skipped || executing_queue_count > 0);
         }
         //drop(to_next_stage);
-        let pair = std::sync::Arc::new((std::sync::Mutex::new(false), std::sync::Condvar::new()));
+        let pair = std::sync::Arc::new((std::sync::Mutex::new(10), std::sync::Condvar::new()));
         to_next_stage.send(ExaminablePayload(Flushable::Flush(pair))).unwrap();
         drop(ee_sender);
         drop(task_sender);

From a7bf7b3c30d66d8e9b8fd960af11bed3f3579648 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 15 Sep 2022 16:29:28 +0900
Subject: [PATCH 2589/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index ea1ff70203c8d4..65eedc9cf1bb86 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1461,7 +1461,7 @@ impl ScheduleStage {
                         match a {
                             Flushable::Flush(pair) => {
                                 let (lock, cvar) = &*pair;
-                                cvar.wait_while(lock.lock().unwrap(), |remaining_threads| remaining_threads == 0).unwrap();
+                                cvar.wait_while(lock.lock().unwrap(), |&remaining_threads| remaining_threads == 0).unwrap();
 
                                 continue;
                             },

From 80a4581febce0f29ab98acdf55ed7af416d999d5 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 15 Sep 2022 16:29:40 +0900
Subject: [PATCH 2590/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 65eedc9cf1bb86..ef4180b904531d 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1461,7 +1461,7 @@ impl ScheduleStage {
                         match a {
                             Flushable::Flush(pair) => {
                                 let (lock, cvar) = &*pair;
-                                cvar.wait_while(lock.lock().unwrap(), |&remaining_threads| remaining_threads == 0).unwrap();
+                                cvar.wait_while(lock.lock().unwrap(), |&mut remaining_threads| remaining_threads == 0).unwrap();
 
                                 continue;
                             },

From e42802b5a3cf694466a09559ac2a1d90f81be113 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 15 Sep 2022 16:33:06 +0900
Subject: [PATCH 2591/3199] save

---
 scheduler/src/lib.rs | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index ef4180b904531d..dfe31f6a116063 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1743,7 +1743,10 @@ pub struct ExecutablePayload(pub Box<ExecutionEnvironment>);
 pub struct UnlockablePayload(pub Box<ExecutionEnvironment>);
 pub struct ExaminablePayload(pub Flushable<Box<ExecutionEnvironment>>);
 
+pub struct FlushContext(std::sync::Mutex<usize>, std::sync::Condvar);
+
+
 pub enum Flushable<T> {
     Payload(T),
-    Flush(std::sync::Arc<(std::sync::Mutex<usize>, std::sync::Condvar)>),
+    Flush(std::sync::Arc<FlushContext>),
 }

From 8942db634cd395ad525817cb98e44392f557eae1 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 15 Sep 2022 16:33:19 +0900
Subject: [PATCH 2592/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index dfe31f6a116063..9b058e4f649c51 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1460,7 +1460,7 @@ impl ScheduleStage {
                     while let Ok(ExaminablePayload(a)) = ee_receiver.recv() {
                         match a {
                             Flushable::Flush(pair) => {
-                                let (lock, cvar) = &*pair;
+                                let FlushContext(lock, cvar) = &*pair;
                                 cvar.wait_while(lock.lock().unwrap(), |&mut remaining_threads| remaining_threads == 0).unwrap();
 
                                 continue;

From 856531c3c496336b5733ac28fc24de17213056a6 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 15 Sep 2022 16:33:43 +0900
Subject: [PATCH 2593/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 9b058e4f649c51..77d4a600b95454 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1691,7 +1691,7 @@ impl ScheduleStage {
             assert!(!select_skipped || executing_queue_count > 0);
         }
         //drop(to_next_stage);
-        let pair = std::sync::Arc::new((std::sync::Mutex::new(10), std::sync::Condvar::new()));
+        let pair = FlushContext(std::sync::Arc::new((std::sync::Mutex::new(10), std::sync::Condvar::new())));
         to_next_stage.send(ExaminablePayload(Flushable::Flush(pair))).unwrap();
         drop(ee_sender);
         drop(task_sender);

From 5d5744e3bcd67f20e0d4079947907a0dbf8c36e9 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 15 Sep 2022 16:34:19 +0900
Subject: [PATCH 2594/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 77d4a600b95454..cf5458b7234324 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1691,7 +1691,7 @@ impl ScheduleStage {
             assert!(!select_skipped || executing_queue_count > 0);
         }
         //drop(to_next_stage);
-        let pair = FlushContext(std::sync::Arc::new((std::sync::Mutex::new(10), std::sync::Condvar::new())));
+        let pair = std::sync::Arc::new(FlushContext((std::sync::Mutex::new(10), std::sync::Condvar::new())));
         to_next_stage.send(ExaminablePayload(Flushable::Flush(pair))).unwrap();
         drop(ee_sender);
         drop(task_sender);

From c607fea37c4b8945e50a79e71122acec30cf9f8e Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 15 Sep 2022 16:34:45 +0900
Subject: [PATCH 2595/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index cf5458b7234324..67e1befc29acee 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1691,7 +1691,7 @@ impl ScheduleStage {
             assert!(!select_skipped || executing_queue_count > 0);
         }
         //drop(to_next_stage);
-        let pair = std::sync::Arc::new(FlushContext((std::sync::Mutex::new(10), std::sync::Condvar::new())));
+        let pair = std::sync::Arc::new(FlushContext(std::sync::Mutex::new(10), std::sync::Condvar::new()));
         to_next_stage.send(ExaminablePayload(Flushable::Flush(pair))).unwrap();
         drop(ee_sender);
         drop(task_sender);

From 535d4a09bf394f2e50498140dc13200d8edf6aff Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 15 Sep 2022 16:36:14 +0900
Subject: [PATCH 2596/3199] save

---
 scheduler/src/lib.rs | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 67e1befc29acee..d5d760a2f67239 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1745,6 +1745,17 @@ pub struct ExaminablePayload(pub Flushable<Box<ExecutionEnvironment>>);
 
 pub struct FlushContext(std::sync::Mutex<usize>, std::sync::Condvar);
 
+impl FlushContext {
+    fn wait() {
+        count -= 1;
+        if count == 0 {
+            notify_all();
+        } else {
+            wait();
+        }
+    }
+}
+
 
 pub enum Flushable<T> {
     Payload(T),

From db62c643c2fed7e4d299a7cb04e8478c26e46ff7 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 15 Sep 2022 21:31:35 +0900
Subject: [PATCH 2597/3199] save

---
 scheduler/src/lib.rs | 21 ++++++++++++---------
 1 file changed, 12 insertions(+), 9 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index d5d760a2f67239..39fc6ab89c57ed 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1460,7 +1460,7 @@ impl ScheduleStage {
                     while let Ok(ExaminablePayload(a)) = ee_receiver.recv() {
                         match a {
                             Flushable::Flush(pair) => {
-                                let FlushContext(lock, cvar) = &*pair;
+                                let Checkpoint(lock, cvar) = &*pair;
                                 cvar.wait_while(lock.lock().unwrap(), |&mut remaining_threads| remaining_threads == 0).unwrap();
 
                                 continue;
@@ -1691,7 +1691,7 @@ impl ScheduleStage {
             assert!(!select_skipped || executing_queue_count > 0);
         }
         //drop(to_next_stage);
-        let pair = std::sync::Arc::new(FlushContext(std::sync::Mutex::new(10), std::sync::Condvar::new()));
+        let pair = std::sync::Arc::new(Checkpoint(std::sync::Mutex::new(10), std::sync::Condvar::new()));
         to_next_stage.send(ExaminablePayload(Flushable::Flush(pair))).unwrap();
         drop(ee_sender);
         drop(task_sender);
@@ -1743,15 +1743,18 @@ pub struct ExecutablePayload(pub Box<ExecutionEnvironment>);
 pub struct UnlockablePayload(pub Box<ExecutionEnvironment>);
 pub struct ExaminablePayload(pub Flushable<Box<ExecutionEnvironment>>);
 
-pub struct FlushContext(std::sync::Mutex<usize>, std::sync::Condvar);
+pub struct Checkpoint(std::sync::Mutex<usize>, std::sync::Condvar);
+
+impl Checkpoint {
+    fn wait_for_restart(&self) {
+        let count = self.0.lock().unwrap();
+
+        *count -= 1;
 
-impl FlushContext {
-    fn wait() {
-        count -= 1;
         if count == 0 {
-            notify_all();
+            self.1.notify_all().unwrap();
         } else {
-            wait();
+            self.0.wait_while(lock.lock().unwrap(), |&mut remaining_threads| remaining_threads == 0).unwrap();
         }
     }
 }
@@ -1759,5 +1762,5 @@ impl FlushContext {
 
 pub enum Flushable<T> {
     Payload(T),
-    Flush(std::sync::Arc<FlushContext>),
+    Flush(std::sync::Arc<Checkpoint>),
 }

From 7921f4d67a58e7e404babc07d1906a2bec9f8b7d Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 15 Sep 2022 21:32:13 +0900
Subject: [PATCH 2598/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 39fc6ab89c57ed..cc78ba2479206c 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1754,7 +1754,7 @@ impl Checkpoint {
         if count == 0 {
             self.1.notify_all().unwrap();
         } else {
-            self.0.wait_while(lock.lock().unwrap(), |&mut remaining_threads| remaining_threads == 0).unwrap();
+            self.0.wait_while(count, |&mut remaining_threads| remaining_threads == 0).unwrap();
         }
     }
 }

From 0c8252ccc787568feb0c992f76cac8e996070c22 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 15 Sep 2022 21:32:22 +0900
Subject: [PATCH 2599/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index cc78ba2479206c..589d038ddc07c7 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1751,7 +1751,7 @@ impl Checkpoint {
 
         *count -= 1;
 
-        if count == 0 {
+        if *count == 0 {
             self.1.notify_all().unwrap();
         } else {
             self.0.wait_while(count, |&mut remaining_threads| remaining_threads == 0).unwrap();

From 6a35ac44d6d078c97dd2567ad19e5cb3154bcac7 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 15 Sep 2022 21:32:33 +0900
Subject: [PATCH 2600/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 589d038ddc07c7..9a80fb21b48783 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1752,7 +1752,7 @@ impl Checkpoint {
         *count -= 1;
 
         if *count == 0 {
-            self.1.notify_all().unwrap();
+            self.1.notify_all();
         } else {
             self.0.wait_while(count, |&mut remaining_threads| remaining_threads == 0).unwrap();
         }

From 5d6f3aabe2364019fa8800036bf3bbf1af24275a Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 15 Sep 2022 21:32:43 +0900
Subject: [PATCH 2601/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 9a80fb21b48783..8ec7f6d047e7e8 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1754,7 +1754,7 @@ impl Checkpoint {
         if *count == 0 {
             self.1.notify_all();
         } else {
-            self.0.wait_while(count, |&mut remaining_threads| remaining_threads == 0).unwrap();
+            self.1.wait_while(count, |&mut remaining_threads| remaining_threads == 0).unwrap();
         }
     }
 }

From 050b8688906732b79211438662bb4cdf6a23aef4 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 15 Sep 2022 21:32:55 +0900
Subject: [PATCH 2602/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 8ec7f6d047e7e8..c2f6982b7d40dc 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1747,7 +1747,7 @@ pub struct Checkpoint(std::sync::Mutex<usize>, std::sync::Condvar);
 
 impl Checkpoint {
     fn wait_for_restart(&self) {
-        let count = self.0.lock().unwrap();
+        let mut count = self.0.lock().unwrap();
 
         *count -= 1;
 

From e0098cdcca2f8d08991d32f0d4e4085a94542c3f Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 15 Sep 2022 21:33:49 +0900
Subject: [PATCH 2603/3199] save

---
 scheduler/src/lib.rs | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index c2f6982b7d40dc..45c1f5f1c6c7df 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1459,10 +1459,8 @@ impl ScheduleStage {
 
                     while let Ok(ExaminablePayload(a)) = ee_receiver.recv() {
                         match a {
-                            Flushable::Flush(pair) => {
-                                let Checkpoint(lock, cvar) = &*pair;
-                                cvar.wait_while(lock.lock().unwrap(), |&mut remaining_threads| remaining_threads == 0).unwrap();
-
+                            Flushable::Flush(checkpoint) => {
+                                checkpoint.wait_for_restart();
                                 continue;
                             },
                             Flushable::Payload(a) => {

From 5f96ed225013dcfc5f96e7f9d4bc94b15d312585 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 15 Sep 2022 21:35:01 +0900
Subject: [PATCH 2604/3199] save

---
 scheduler/src/lib.rs | 1 +
 1 file changed, 1 insertion(+)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 45c1f5f1c6c7df..36ea45b01ae8c1 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1449,6 +1449,7 @@ impl ScheduleStage {
         let (to_next_stage, maybe_reaper_thread_handle) = if let Some(to_next_stage) = maybe_to_next_stage {
             (to_next_stage, None)
         } else {
+            todo!();
             let h = std::thread::Builder::new()
                 .name("solScReaper".to_string())
                 .spawn(move || {

From 6bfca267a22bae2cb5e56819e07294de4fd17c22 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 15 Sep 2022 21:35:43 +0900
Subject: [PATCH 2605/3199] save

---
 scheduler/src/lib.rs | 1 +
 1 file changed, 1 insertion(+)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 36ea45b01ae8c1..9ab4531702602c 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1751,6 +1751,7 @@ impl Checkpoint {
         *count -= 1;
 
         if *count == 0 {
+            drop(count);
             self.1.notify_all();
         } else {
             self.1.wait_while(count, |&mut remaining_threads| remaining_threads == 0).unwrap();

From 0dd452b411088ba85c3702510dbbe6d5f5f8e09f Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 15 Sep 2022 21:36:28 +0900
Subject: [PATCH 2606/3199] save

---
 scheduler/src/lib.rs | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 9ab4531702602c..b9d7be7a7a7ea9 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1746,15 +1746,15 @@ pub struct Checkpoint(std::sync::Mutex<usize>, std::sync::Condvar);
 
 impl Checkpoint {
     fn wait_for_restart(&self) {
-        let mut count = self.0.lock().unwrap();
+        let mut remaining_threads = self.0.lock().unwrap();
 
-        *count -= 1;
+        *remaining_threads -= 1;
 
-        if *count == 0 {
-            drop(count);
+        if *remaining_threads == 0 {
+            drop(remaining_threads);
             self.1.notify_all();
         } else {
-            self.1.wait_while(count, |&mut remaining_threads| remaining_threads == 0).unwrap();
+            self.1.wait_while(remaining_threads, |&mut remaining_threads| remaining_threads == 0).unwrap();
         }
     }
 }

From e2c508fea5cc49802dacae4756c587762d201769 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 15 Sep 2022 21:37:21 +0900
Subject: [PATCH 2607/3199] save

---
 scheduler/src/lib.rs | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index b9d7be7a7a7ea9..2f602202af8adb 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1746,15 +1746,15 @@ pub struct Checkpoint(std::sync::Mutex<usize>, std::sync::Condvar);
 
 impl Checkpoint {
     fn wait_for_restart(&self) {
-        let mut remaining_threads = self.0.lock().unwrap();
+        let mut remaining_threads_guard = self.0.lock().unwrap();
 
-        *remaining_threads -= 1;
+        *remaining_threads_guard -= 1;
 
-        if *remaining_threads == 0 {
-            drop(remaining_threads);
+        if *remaining_threads_guard == 0 {
+            drop(remaining_threads_guard);
             self.1.notify_all();
         } else {
-            self.1.wait_while(remaining_threads, |&mut remaining_threads| remaining_threads == 0).unwrap();
+            self.1.wait_while(remaining_threads_guard, |&mut remaining_threads| remaining_threads == 0).unwrap();
         }
     }
 }

From 934bd404db63f04a431825d06f248d19acfd85bf Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 15 Sep 2022 21:41:31 +0900
Subject: [PATCH 2608/3199] save

---
 runtime/src/bank.rs  | 2 ++
 scheduler/src/lib.rs | 4 +---
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 47023a4e5c9f4d..1c6a0f0b5a66ef 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1434,6 +1434,8 @@ impl Default for Scheduler {
             drop(transaction_receiver);
             drop(scheduled_ee_sender);
             drop(processed_ee_receiver);
+            let checkpoint = std::sync::Arc::new(Checkpoint(std::sync::Mutex::new(10), std::sync::Condvar::new()));
+            retired_ee_sender.send(ExaminablePayload(Flushable::Flush(checkpoint))).unwrap();
 
             Ok((started.0.elapsed(), started.1.elapsed()))
         }).unwrap();
diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 2f602202af8adb..b1804f0e1f6a8c 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1689,9 +1689,7 @@ impl ScheduleStage {
             }
             assert!(!select_skipped || executing_queue_count > 0);
         }
-        //drop(to_next_stage);
-        let pair = std::sync::Arc::new(Checkpoint(std::sync::Mutex::new(10), std::sync::Condvar::new()));
-        to_next_stage.send(ExaminablePayload(Flushable::Flush(pair))).unwrap();
+        drop(to_next_stage);
         drop(ee_sender);
         drop(task_sender);
         drop(task_receiver);

From e9afa17df2445fbd1b18ba83006046bf1188c06f Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 15 Sep 2022 21:41:50 +0900
Subject: [PATCH 2609/3199] save

---
 runtime/src/bank.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 1c6a0f0b5a66ef..efc80f479c59be 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1434,7 +1434,7 @@ impl Default for Scheduler {
             drop(transaction_receiver);
             drop(scheduled_ee_sender);
             drop(processed_ee_receiver);
-            let checkpoint = std::sync::Arc::new(Checkpoint(std::sync::Mutex::new(10), std::sync::Condvar::new()));
+            let checkpoint = std::sync::Arc::new(solana_scheduler::Checkpoint(std::sync::Mutex::new(10), std::sync::Condvar::new()));
             retired_ee_sender.send(ExaminablePayload(Flushable::Flush(checkpoint))).unwrap();
 
             Ok((started.0.elapsed(), started.1.elapsed()))

From a6098983d0c0d74e8ecbb9aa0ab7580aa3c6c5cb Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 15 Sep 2022 21:42:05 +0900
Subject: [PATCH 2610/3199] save

---
 runtime/src/bank.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index efc80f479c59be..12d3ea2f661cb3 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1435,7 +1435,7 @@ impl Default for Scheduler {
             drop(scheduled_ee_sender);
             drop(processed_ee_receiver);
             let checkpoint = std::sync::Arc::new(solana_scheduler::Checkpoint(std::sync::Mutex::new(10), std::sync::Condvar::new()));
-            retired_ee_sender.send(ExaminablePayload(Flushable::Flush(checkpoint))).unwrap();
+            retired_ee_sender.send(ExaminablePayload(solana_scheduler::Flushable::Flush(checkpoint))).unwrap();
 
             Ok((started.0.elapsed(), started.1.elapsed()))
         }).unwrap();

From 1d8ed17545c01e1c7566dca36ede45fa63d96686 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 15 Sep 2022 21:42:18 +0900
Subject: [PATCH 2611/3199] save

---
 runtime/src/bank.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 12d3ea2f661cb3..07d3309bcfc17d 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1435,7 +1435,7 @@ impl Default for Scheduler {
             drop(scheduled_ee_sender);
             drop(processed_ee_receiver);
             let checkpoint = std::sync::Arc::new(solana_scheduler::Checkpoint(std::sync::Mutex::new(10), std::sync::Condvar::new()));
-            retired_ee_sender.send(ExaminablePayload(solana_scheduler::Flushable::Flush(checkpoint))).unwrap();
+            retired_ee_sender.send(solana_scheduler::ExaminablePayload(solana_scheduler::Flushable::Flush(checkpoint))).unwrap();
 
             Ok((started.0.elapsed(), started.1.elapsed()))
         }).unwrap();

From 030d922cad275712265f9bb46a57e431f9f7bad3 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 15 Sep 2022 21:42:47 +0900
Subject: [PATCH 2612/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index b1804f0e1f6a8c..f4a3a41d131e77 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1740,7 +1740,7 @@ pub struct ExecutablePayload(pub Box<ExecutionEnvironment>);
 pub struct UnlockablePayload(pub Box<ExecutionEnvironment>);
 pub struct ExaminablePayload(pub Flushable<Box<ExecutionEnvironment>>);
 
-pub struct Checkpoint(std::sync::Mutex<usize>, std::sync::Condvar);
+pub struct Checkpoint(pub std::sync::Mutex<usize>, pub std::sync::Condvar);
 
 impl Checkpoint {
     fn wait_for_restart(&self) {

From 0b6e38c6061169f1f05d6f12c3add4daa250c25e Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 15 Sep 2022 21:44:15 +0900
Subject: [PATCH 2613/3199] save

---
 runtime/src/bank.rs  | 2 +-
 scheduler/src/lib.rs | 4 ++++
 2 files changed, 5 insertions(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 07d3309bcfc17d..0af85e218f76bb 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1434,7 +1434,7 @@ impl Default for Scheduler {
             drop(transaction_receiver);
             drop(scheduled_ee_sender);
             drop(processed_ee_receiver);
-            let checkpoint = std::sync::Arc::new(solana_scheduler::Checkpoint(std::sync::Mutex::new(10), std::sync::Condvar::new()));
+            let checkpoint = solana_scheduler::Checkpoint::new(3);
             retired_ee_sender.send(solana_scheduler::ExaminablePayload(solana_scheduler::Flushable::Flush(checkpoint))).unwrap();
 
             Ok((started.0.elapsed(), started.1.elapsed()))
diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index f4a3a41d131e77..2f83da13c2c8ef 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1755,6 +1755,10 @@ impl Checkpoint {
             self.1.wait_while(remaining_threads_guard, |&mut remaining_threads| remaining_threads == 0).unwrap();
         }
     }
+
+    fn new(remaining_threads: usize) -> std::sync::Arc<Self> {
+        std::sync::Arc::new(solana_scheduler::Checkpoint(std::sync::Mutex::new(remaining_threads), std::sync::Condvar::new()))
+    }
 }
 
 

From d89d47783bda0e7f7a57ff1fc971f467401bc248 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 15 Sep 2022 21:44:28 +0900
Subject: [PATCH 2614/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 2f83da13c2c8ef..a5a54d4d07b523 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1757,7 +1757,7 @@ impl Checkpoint {
     }
 
     fn new(remaining_threads: usize) -> std::sync::Arc<Self> {
-        std::sync::Arc::new(solana_scheduler::Checkpoint(std::sync::Mutex::new(remaining_threads), std::sync::Condvar::new()))
+        std::sync::Arc::new(Self(std::sync::Mutex::new(remaining_threads), std::sync::Condvar::new()))
     }
 }
 

From 1110c711c171cb960f9c13da5d29b2dbe2b68ba0 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 15 Sep 2022 21:44:38 +0900
Subject: [PATCH 2615/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index a5a54d4d07b523..0f27f860e112b2 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1756,7 +1756,7 @@ impl Checkpoint {
         }
     }
 
-    fn new(remaining_threads: usize) -> std::sync::Arc<Self> {
+    pub fn new(remaining_threads: usize) -> std::sync::Arc<Self> {
         std::sync::Arc::new(Self(std::sync::Mutex::new(remaining_threads), std::sync::Condvar::new()))
     }
 }

From 481522b03ca56eabf160a11cd667a863f7c8113e Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 15 Sep 2022 21:45:04 +0900
Subject: [PATCH 2616/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 0f27f860e112b2..019fd9db952e7c 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1752,7 +1752,7 @@ impl Checkpoint {
             drop(remaining_threads_guard);
             self.1.notify_all();
         } else {
-            self.1.wait_while(remaining_threads_guard, |&mut remaining_threads| remaining_threads == 0).unwrap();
+            self.1.wait_while(remaining_threads_guard, |&mut remaining_threads| remaining_threads == 0).unwrap().unwrap();
         }
     }
 

From 309eed63b2284e2de216b25602650c9e5a7a24fd Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 15 Sep 2022 21:45:16 +0900
Subject: [PATCH 2617/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 019fd9db952e7c..0f27f860e112b2 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1752,7 +1752,7 @@ impl Checkpoint {
             drop(remaining_threads_guard);
             self.1.notify_all();
         } else {
-            self.1.wait_while(remaining_threads_guard, |&mut remaining_threads| remaining_threads == 0).unwrap().unwrap();
+            self.1.wait_while(remaining_threads_guard, |&mut remaining_threads| remaining_threads == 0).unwrap();
         }
     }
 

From 96405424fc119beb9ea970c99a99d6649b4a1ae4 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 15 Sep 2022 21:45:45 +0900
Subject: [PATCH 2618/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 0f27f860e112b2..29ff14ec4d1757 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1752,7 +1752,7 @@ impl Checkpoint {
             drop(remaining_threads_guard);
             self.1.notify_all();
         } else {
-            self.1.wait_while(remaining_threads_guard, |&mut remaining_threads| remaining_threads == 0).unwrap();
+            self.1.wait_while(remaining_threads_guard, |&mut remaining_threads| remaining_threads == 0).unwrap()
         }
     }
 

From 80e14ced9d6e4f18ac6e97885890c31415ebbbeb Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 15 Sep 2022 21:45:57 +0900
Subject: [PATCH 2619/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 29ff14ec4d1757..19ccbb235a62ca 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1752,7 +1752,7 @@ impl Checkpoint {
             drop(remaining_threads_guard);
             self.1.notify_all();
         } else {
-            self.1.wait_while(remaining_threads_guard, |&mut remaining_threads| remaining_threads == 0).unwrap()
+            *self.1.wait_while(remaining_threads_guard, |&mut remaining_threads| remaining_threads == 0).unwrap();
         }
     }
 

From f7a56a7d0f975239864c87e97857fc182f072da1 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 15 Sep 2022 21:46:13 +0900
Subject: [PATCH 2620/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 19ccbb235a62ca..09bf10fc35f41d 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1752,7 +1752,7 @@ impl Checkpoint {
             drop(remaining_threads_guard);
             self.1.notify_all();
         } else {
-            *self.1.wait_while(remaining_threads_guard, |&mut remaining_threads| remaining_threads == 0).unwrap();
+            let _ = *self.1.wait_while(remaining_threads_guard, |&mut remaining_threads| remaining_threads == 0).unwrap();
         }
     }
 

From c875e8297f723babc382eacad55983b336129d17 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 15 Sep 2022 21:50:53 +0900
Subject: [PATCH 2621/3199] save

---
 runtime/src/bank.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 0af85e218f76bb..98d5a18c047a76 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1434,8 +1434,8 @@ impl Default for Scheduler {
             drop(transaction_receiver);
             drop(scheduled_ee_sender);
             drop(processed_ee_receiver);
-            let checkpoint = solana_scheduler::Checkpoint::new(3);
-            retired_ee_sender.send(solana_scheduler::ExaminablePayload(solana_scheduler::Flushable::Flush(checkpoint))).unwrap();
+            //let checkpoint = solana_scheduler::Checkpoint::new(3);
+            //retired_ee_sender.send(solana_scheduler::ExaminablePayload(solana_scheduler::Flushable::Flush(checkpoint))).unwrap();
 
             Ok((started.0.elapsed(), started.1.elapsed()))
         }).unwrap();

From f9be59e2214cddc9469c39ca17ada64952f48ee2 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 15 Sep 2022 21:52:49 +0900
Subject: [PATCH 2622/3199] save

---
 runtime/src/bank.rs | 25 ++++++++++++++++---------
 1 file changed, 16 insertions(+), 9 deletions(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 98d5a18c047a76..61935eef731b4d 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1393,16 +1393,23 @@ impl Default for Scheduler {
                 thread_priority::set_current_thread_priority(thread_priority::ThreadPriority::Max).unwrap();
             }
 
-            while let Ok(solana_scheduler::ExaminablePayload(mut ee)) = retired_ee_receiver.recv() {
-                if ee.is_aborted() {
-                    warn!(
-                        "scheduler: Unexpected validator error: {:?}, transaction: {:?}",
-                        ee.execution_result, ee.task.tx.0
-                    );
-                    collected_errors_in_collector_thread.lock().unwrap().push(ee.execution_result.take().unwrap());
+            while let Ok(solana_scheduler::ExaminablePayload(a)) = retired_ee_receiver.recv() {
+                match a {
+                    solana_scheduler::Flushable::Flush(checkpoint) => {
+                        checkpoint.wait_for_restart();
+                        continue;
+                    },
+                    solana_scheduler::Flushable::Payload(mut ee) => {
+                        if ee.is_aborted() {
+                            warn!(
+                                "scheduler: Unexpected validator error: {:?}, transaction: {:?}",
+                                ee.execution_result, ee.task.tx.0
+                            );
+                            collected_errors_in_collector_thread.lock().unwrap().push(ee.execution_result.take().unwrap());
+                        }
+                        drop(ee);
+                    },
                 }
-
-                drop(ee);
             }
             Ok((started.0.elapsed(), started.1.elapsed()))
         }).unwrap();

From 4305d534d6a536853a19503a6d5a8503f0d8e6bd Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 15 Sep 2022 21:53:10 +0900
Subject: [PATCH 2623/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 09bf10fc35f41d..c9ebeee0ce4b06 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1743,7 +1743,7 @@ pub struct ExaminablePayload(pub Flushable<Box<ExecutionEnvironment>>);
 pub struct Checkpoint(pub std::sync::Mutex<usize>, pub std::sync::Condvar);
 
 impl Checkpoint {
-    fn wait_for_restart(&self) {
+    pub fn wait_for_restart(&self) {
         let mut remaining_threads_guard = self.0.lock().unwrap();
 
         *remaining_threads_guard -= 1;

From 7b75d077cd952543688c4a3d2b1f66fd118bb7f1 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 15 Sep 2022 21:56:19 +0900
Subject: [PATCH 2624/3199] save

---
 runtime/src/bank.rs | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 61935eef731b4d..1d9bcf21c4093a 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1411,6 +1411,7 @@ impl Default for Scheduler {
                     },
                 }
             }
+
             Ok((started.0.elapsed(), started.1.elapsed()))
         }).unwrap();
 
@@ -1441,8 +1442,6 @@ impl Default for Scheduler {
             drop(transaction_receiver);
             drop(scheduled_ee_sender);
             drop(processed_ee_receiver);
-            //let checkpoint = solana_scheduler::Checkpoint::new(3);
-            //retired_ee_sender.send(solana_scheduler::ExaminablePayload(solana_scheduler::Flushable::Flush(checkpoint))).unwrap();
 
             Ok((started.0.elapsed(), started.1.elapsed()))
         }).unwrap();
@@ -1475,7 +1474,11 @@ impl Scheduler {
 
         info!("Scheduler::gracefully_stop(): id_{:016x} waiting..", self.random_id);
         let transaction_sender = self.transaction_sender.take().unwrap();
-        drop(transaction_sender);
+
+        //drop(transaction_sender);
+        let checkpoint = solana_scheduler::Checkpoint::new(3);
+        transaction_sender.send(solana_scheduler::RunnablePayload(solana_scheduler::Flushable::Flush(checkpoint))).unwrap();
+
         let executing_thread_duration_pairs: Result<Vec<_>> = self.executing_thread_handles.take().unwrap().into_iter().map(|executing_thread_handle| {
             executing_thread_handle.join().unwrap().map(|u| (u.0.as_micros(), u.1.as_micros()))
         }).collect();

From 89b5463f2f616070650fd82be1e7fc823f83ff59 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 15 Sep 2022 21:56:43 +0900
Subject: [PATCH 2625/3199] save

---
 runtime/src/bank.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 1d9bcf21c4093a..0e27b1606f914b 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1477,7 +1477,7 @@ impl Scheduler {
 
         //drop(transaction_sender);
         let checkpoint = solana_scheduler::Checkpoint::new(3);
-        transaction_sender.send(solana_scheduler::RunnablePayload(solana_scheduler::Flushable::Flush(checkpoint))).unwrap();
+        transaction_sender.send(solana_scheduler::SchedulablePayload(solana_scheduler::Flushable::Flush(checkpoint))).unwrap();
 
         let executing_thread_duration_pairs: Result<Vec<_>> = self.executing_thread_handles.take().unwrap().into_iter().map(|executing_thread_handle| {
             executing_thread_handle.join().unwrap().map(|u| (u.0.as_micros(), u.1.as_micros()))

From 9af365fe7b49e385d9f56c136ec973726295b72c Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 15 Sep 2022 21:57:13 +0900
Subject: [PATCH 2626/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index c9ebeee0ce4b06..d53222e971ad70 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1735,7 +1735,7 @@ impl ScheduleStage {
     }
 }
 
-pub struct SchedulablePayload(pub TaskInQueue);
+pub struct SchedulablePayload(pub Flashable<TaskInQueue>);
 pub struct ExecutablePayload(pub Box<ExecutionEnvironment>);
 pub struct UnlockablePayload(pub Box<ExecutionEnvironment>);
 pub struct ExaminablePayload(pub Flushable<Box<ExecutionEnvironment>>);

From b41e044aa2cf0c634d5fd20f89d0cb6c2d3db982 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 15 Sep 2022 21:57:21 +0900
Subject: [PATCH 2627/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index d53222e971ad70..25de2ece89395d 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1735,7 +1735,7 @@ impl ScheduleStage {
     }
 }
 
-pub struct SchedulablePayload(pub Flashable<TaskInQueue>);
+pub struct SchedulablePayload(pub Flushable<TaskInQueue>);
 pub struct ExecutablePayload(pub Box<ExecutionEnvironment>);
 pub struct UnlockablePayload(pub Box<ExecutionEnvironment>);
 pub struct ExaminablePayload(pub Flushable<Box<ExecutionEnvironment>>);

From 6f742429d2520cef05dbacfd14dcc19511c94c1d Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 15 Sep 2022 21:58:59 +0900
Subject: [PATCH 2628/3199] save

---
 scheduler/src/lib.rs | 12 ++++++++----
 1 file changed, 8 insertions(+), 4 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 25de2ece89395d..d2dc999002a282 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1680,10 +1680,14 @@ impl ScheduleStage {
                         to_next_stage.send(ExaminablePayload(Flushable::Payload(processed_execution_environment))).unwrap();
                     }
                     if !empty_from {
-                        let task = from_prev.recv().unwrap().0;
-                        from_len = from_len.checked_sub(1).unwrap();
-                        empty_from = from_len == 0;
-                        Self::register_runnable_task(task, runnable_queue, &mut sequence_time);
+                        let schedulable = from_prev.recv().unwrap();
+                        match schedulable {
+                            Flushable::Payload(task) => {
+                                from_len = from_len.checked_sub(1).unwrap();
+                                empty_from = from_len == 0;
+                                Self::register_runnable_task(task, runnable_queue, &mut sequence_time);
+                            },
+                        }
                     }
                 }
             }

From 9cc4cefbe4f56d3ff9b3c4b9755315f25410b465 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 15 Sep 2022 21:59:22 +0900
Subject: [PATCH 2629/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index d2dc999002a282..bfea3d604a303d 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1680,7 +1680,7 @@ impl ScheduleStage {
                         to_next_stage.send(ExaminablePayload(Flushable::Payload(processed_execution_environment))).unwrap();
                     }
                     if !empty_from {
-                        let schedulable = from_prev.recv().unwrap();
+                        let SchedulablePayload(schedulable) = from_prev.recv().unwrap();
                         match schedulable {
                             Flushable::Payload(task) => {
                                 from_len = from_len.checked_sub(1).unwrap();

From 4acd904f5d76204b01148a51042e0a6a25909fee Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 15 Sep 2022 22:01:12 +0900
Subject: [PATCH 2630/3199] save

---
 scheduler/src/lib.rs | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index bfea3d604a303d..a26c213044e863 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1682,6 +1682,8 @@ impl ScheduleStage {
                     if !empty_from {
                         let SchedulablePayload(schedulable) = from_prev.recv().unwrap();
                         match schedulable {
+                            Flushable::Flush(checkpoint) => {
+                            },
                             Flushable::Payload(task) => {
                                 from_len = from_len.checked_sub(1).unwrap();
                                 empty_from = from_len == 0;

From 9f8b76c25d5b19dade5db9b75bdc7f8e1fb56f9a Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 15 Sep 2022 22:02:13 +0900
Subject: [PATCH 2631/3199] save

---
 scheduler/src/lib.rs | 1 +
 1 file changed, 1 insertion(+)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index a26c213044e863..00b48918bcd9d7 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1685,6 +1685,7 @@ impl ScheduleStage {
                             Flushable::Flush(checkpoint) => {
                             },
                             Flushable::Payload(task) => {
+                                let u: usize = task;
                                 from_len = from_len.checked_sub(1).unwrap();
                                 empty_from = from_len == 0;
                                 Self::register_runnable_task(task, runnable_queue, &mut sequence_time);

From ec8edc29fab1c8e64d6f0e24bfbeef2d95da9920 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 15 Sep 2022 22:05:51 +0900
Subject: [PATCH 2632/3199] save

---
 scheduler/src/lib.rs | 10 +++++++---
 1 file changed, 7 insertions(+), 3 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 00b48918bcd9d7..959581de2c9b70 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1681,13 +1681,17 @@ impl ScheduleStage {
                     }
                     if !empty_from {
                         let SchedulablePayload(schedulable) = from_prev.recv().unwrap();
+                        from_len = from_len.checked_sub(1).unwrap();
+                        empty_from = from_len == 0;
                         match schedulable {
                             Flushable::Flush(checkpoint) => {
+                               assert!(empty_from);
+                               assert_eq!(from_prev.len(), 0);
+                               assert!(!from_disconnected);
+                               from_disconnected = true;
+                               from_prev = never;
                             },
                             Flushable::Payload(task) => {
-                                let u: usize = task;
-                                from_len = from_len.checked_sub(1).unwrap();
-                                empty_from = from_len == 0;
                                 Self::register_runnable_task(task, runnable_queue, &mut sequence_time);
                             },
                         }

From bfd92f8cd8709ebb631f531f52d6b16e85f116fa Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 15 Sep 2022 22:08:30 +0900
Subject: [PATCH 2633/3199] save

---
 scheduler/src/lib.rs | 27 +++++++++++++++++++--------
 1 file changed, 19 insertions(+), 8 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 959581de2c9b70..cac3b97aa8cef8 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1533,14 +1533,25 @@ impl ScheduleStage {
                        }
                    }
                    recv(from_prev) -> maybe_from => {
-                       if let Ok(SchedulablePayload(task)) = maybe_from {
-                           Self::register_runnable_task(task, runnable_queue, &mut sequence_time);
-                       } else {
-                           assert_eq!(from_prev.len(), 0);
-                           assert!(!from_disconnected);
-                           from_disconnected = true;
-                           from_prev = never;
-                           trace!("flushing2..: {:?} {} {} {} {}", (from_disconnected, from_exec_disconnected), runnable_queue.task_count(), contended_count, executing_queue_count, provisioning_tracker_count);
+                       match maybe_from {
+                           Ok(SchedulablePayload(Flushable::Payload(task))) => {
+                                Self::register_runnable_task(task, runnable_queue, &mut sequence_time);
+                           },
+                           Ok(SchedulablePayload(Flushable::Flush(checkpoint_in_payload)) => {
+                               assert_eq!(from_prev.len(), 0);
+                               assert!(!from_disconnected);
+                               from_disconnected = true;
+                               from_prev = never;
+                               trace!("flushing2..: {:?} {} {} {} {}", (from_disconnected, from_exec_disconnected), runnable_queue.task_count(), contended_count, executing_queue_count, provisioning_tracker_count);
+                               checkpoint = checkpoint_in_payload;
+                           },
+                           Err(_) {
+                               assert_eq!(from_prev.len(), 0);
+                               assert!(!from_disconnected);
+                               from_disconnected = true;
+                               from_prev = never;
+                               trace!("flushing2..: {:?} {} {} {} {}", (from_disconnected, from_exec_disconnected), runnable_queue.task_count(), contended_count, executing_queue_count, provisioning_tracker_count);
+                           },
                        }
                    }
                 }

From ac04937311f7746d1c51f193bc10858e10d0b1a7 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 15 Sep 2022 22:08:38 +0900
Subject: [PATCH 2634/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index cac3b97aa8cef8..d5b7abf17a3dab 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1545,7 +1545,7 @@ impl ScheduleStage {
                                trace!("flushing2..: {:?} {} {} {} {}", (from_disconnected, from_exec_disconnected), runnable_queue.task_count(), contended_count, executing_queue_count, provisioning_tracker_count);
                                checkpoint = checkpoint_in_payload;
                            },
-                           Err(_) {
+                           Err(_) => {
                                assert_eq!(from_prev.len(), 0);
                                assert!(!from_disconnected);
                                from_disconnected = true;

From 14b261f99fb61c7e2e7d6b3c21b89d98831457f7 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 15 Sep 2022 22:10:16 +0900
Subject: [PATCH 2635/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index d5b7abf17a3dab..8c00690ac1f471 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1537,7 +1537,7 @@ impl ScheduleStage {
                            Ok(SchedulablePayload(Flushable::Payload(task))) => {
                                 Self::register_runnable_task(task, runnable_queue, &mut sequence_time);
                            },
-                           Ok(SchedulablePayload(Flushable::Flush(checkpoint_in_payload)) => {
+                           Ok(SchedulablePayload(Flushable::Flush(checkpoint_in_payload))) => {
                                assert_eq!(from_prev.len(), 0);
                                assert!(!from_disconnected);
                                from_disconnected = true;

From 85469cb5bbd7d9b0dfe55fb072bde3cdb8bfd45f Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 15 Sep 2022 22:10:42 +0900
Subject: [PATCH 2636/3199] save

---
 scheduler/src/lib.rs | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 8c00690ac1f471..47e3caef5a3a7e 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1512,6 +1512,8 @@ impl ScheduleStage {
 
         let (mut from_disconnected, mut from_exec_disconnected, mut no_more_work): (bool, bool, bool) = Default::default();
 
+        let checkpoint = None;
+
         loop {
             let mut select_skipped = false;
 
@@ -1543,7 +1545,7 @@ impl ScheduleStage {
                                from_disconnected = true;
                                from_prev = never;
                                trace!("flushing2..: {:?} {} {} {} {}", (from_disconnected, from_exec_disconnected), runnable_queue.task_count(), contended_count, executing_queue_count, provisioning_tracker_count);
-                               checkpoint = checkpoint_in_payload;
+                               checkpoint = Some(checkpoint_in_payload);
                            },
                            Err(_) => {
                                assert_eq!(from_prev.len(), 0);

From 8289fac3b62073f1598c9b91bc13484cfc52a5e7 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 15 Sep 2022 22:10:49 +0900
Subject: [PATCH 2637/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 47e3caef5a3a7e..2c7c07c1968b0d 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1512,7 +1512,7 @@ impl ScheduleStage {
 
         let (mut from_disconnected, mut from_exec_disconnected, mut no_more_work): (bool, bool, bool) = Default::default();
 
-        let checkpoint = None;
+        let mut checkpoint = None;
 
         loop {
             let mut select_skipped = false;

From 704a8d2fb6c5b4d40598456a25843ffed81ad6cc Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 15 Sep 2022 22:11:39 +0900
Subject: [PATCH 2638/3199] save

---
 scheduler/src/lib.rs | 1 +
 1 file changed, 1 insertion(+)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 2c7c07c1968b0d..ed0d4b43db1428 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1550,6 +1550,7 @@ impl ScheduleStage {
                            Err(_) => {
                                assert_eq!(from_prev.len(), 0);
                                assert!(!from_disconnected);
+                               assert!(checkpoint.is_none());
                                from_disconnected = true;
                                from_prev = never;
                                trace!("flushing2..: {:?} {} {} {} {}", (from_disconnected, from_exec_disconnected), runnable_queue.task_count(), contended_count, executing_queue_count, provisioning_tracker_count);

From 3b0f5d3d1eab24b0f7ef44ea576be98b1c271c75 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 15 Sep 2022 22:13:14 +0900
Subject: [PATCH 2639/3199] save

---
 scheduler/src/lib.rs | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index ed0d4b43db1428..982a26433f3e59 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1698,12 +1698,13 @@ impl ScheduleStage {
                         from_len = from_len.checked_sub(1).unwrap();
                         empty_from = from_len == 0;
                         match schedulable {
-                            Flushable::Flush(checkpoint) => {
+                            Flushable::Flush(checkpoint_in_payload) => {
                                assert!(empty_from);
                                assert_eq!(from_prev.len(), 0);
                                assert!(!from_disconnected);
                                from_disconnected = true;
                                from_prev = never;
+                               checkpoint = Some(checkpoint_in_payload);
                             },
                             Flushable::Payload(task) => {
                                 Self::register_runnable_task(task, runnable_queue, &mut sequence_time);

From 836238d492aeb31cb1ed50c7ca4fef5a1cc25fd9 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 15 Sep 2022 22:16:46 +0900
Subject: [PATCH 2640/3199] save

---
 scheduler/src/lib.rs | 14 ++++++++------
 1 file changed, 8 insertions(+), 6 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 982a26433f3e59..dc85814b24d78e 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1512,7 +1512,7 @@ impl ScheduleStage {
 
         let (mut from_disconnected, mut from_exec_disconnected, mut no_more_work): (bool, bool, bool) = Default::default();
 
-        let mut checkpoint = None;
+        let mut maybe_checkpoint = None;
 
         loop {
             let mut select_skipped = false;
@@ -1539,18 +1539,18 @@ impl ScheduleStage {
                            Ok(SchedulablePayload(Flushable::Payload(task))) => {
                                 Self::register_runnable_task(task, runnable_queue, &mut sequence_time);
                            },
-                           Ok(SchedulablePayload(Flushable::Flush(checkpoint_in_payload))) => {
+                           Ok(SchedulablePayload(Flushable::Flush(checkpoint))) => {
                                assert_eq!(from_prev.len(), 0);
                                assert!(!from_disconnected);
                                from_disconnected = true;
                                from_prev = never;
                                trace!("flushing2..: {:?} {} {} {} {}", (from_disconnected, from_exec_disconnected), runnable_queue.task_count(), contended_count, executing_queue_count, provisioning_tracker_count);
-                               checkpoint = Some(checkpoint_in_payload);
+                               maybe_checkpoint = Some(checkpoint);
                            },
                            Err(_) => {
                                assert_eq!(from_prev.len(), 0);
                                assert!(!from_disconnected);
-                               assert!(checkpoint.is_none());
+                               assert!(maybe_checkpoint.is_none());
                                from_disconnected = true;
                                from_prev = never;
                                trace!("flushing2..: {:?} {} {} {} {}", (from_disconnected, from_exec_disconnected), runnable_queue.task_count(), contended_count, executing_queue_count, provisioning_tracker_count);
@@ -1698,13 +1698,13 @@ impl ScheduleStage {
                         from_len = from_len.checked_sub(1).unwrap();
                         empty_from = from_len == 0;
                         match schedulable {
-                            Flushable::Flush(checkpoint_in_payload) => {
+                            Flushable::Flush(checkpoint) => {
                                assert!(empty_from);
                                assert_eq!(from_prev.len(), 0);
                                assert!(!from_disconnected);
                                from_disconnected = true;
                                from_prev = never;
-                               checkpoint = Some(checkpoint_in_payload);
+                               maybe_checkpoint = Some(checkpoint);
                             },
                             Flushable::Payload(task) => {
                                 Self::register_runnable_task(task, runnable_queue, &mut sequence_time);
@@ -1719,6 +1719,7 @@ impl ScheduleStage {
         drop(ee_sender);
         drop(task_sender);
         drop(task_receiver);
+
         if let Some(h) = maybe_reaper_thread_handle {
             h.join().unwrap().unwrap();
         }
@@ -1730,6 +1731,7 @@ impl ScheduleStage {
         let elapsed = start_time.elapsed();
         let elapsed2 = elapsed.as_micros();
         info!("schedule_once:final id_{:016x} (no_more_work: {}) ch(prev: {}, exec: {}|{}), runnnable: {}, contended: {}, (immediate+provisional)/max: ({}+{})/{} uncontended: {} stuck: {} miss: {}, overall: {}txs/{}us={}tps!", random_id, no_more_work, (if from_disconnected { "-".to_string() } else { format!("{}", from_prev.len()) }), to_execute_substage.len(), (if from_exec_disconnected { "-".to_string() } else { format!("{}", from_exec.len())}), runnable_queue.task_count(), contended_count, executing_queue_count, provisioning_tracker_count, max_executing_queue_count, address_book.uncontended_task_ids.len(), address_book.stuck_tasks.len(), failed_lock_count, processed_count, elapsed.as_micros(), 1_000_000_u128*(processed_count as u128)/elapsed2);
+        maybe_checkpoint
     }
 
     pub fn run(

From 8e516c5a020d45c442156ba6e0f6ac3e6d093848 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 15 Sep 2022 22:17:27 +0900
Subject: [PATCH 2641/3199] save

---
 scheduler/src/lib.rs | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index dc85814b24d78e..b8ba058cc38b37 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1415,6 +1415,7 @@ impl ScheduleStage {
         Self::push_to_runnable_queue(weighted_tx, runnable_queue)
     }
 
+    #[must_use]
     fn _run<'a, AST: AtScheduleThread>(
         ast: AST,
         random_id: u64,
@@ -1426,7 +1427,7 @@ impl ScheduleStage {
         from_exec: &crossbeam_channel::Receiver<UnlockablePayload>,
         maybe_to_next_stage: Option<&crossbeam_channel::Sender<ExaminablePayload>>, // assume nonblocking
         never: &'a crossbeam_channel::Receiver<SchedulablePayload>,
-    ) {
+    ) -> Option<std::sync::Arc<Checkpoint>> {
         let start_time = std::time::Instant::now();
         let (mut last_time, mut last_processed_count) = (start_time.clone(), 0_usize);
         info!("schedule_once:initial id_{:016x}", random_id);

From 5652c8e3bbd55de8e71b6d3f51c50bc80336a618 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 15 Sep 2022 22:18:02 +0900
Subject: [PATCH 2642/3199] save

---
 scheduler/src/lib.rs | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index b8ba058cc38b37..147664b73a68d4 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1735,6 +1735,7 @@ impl ScheduleStage {
         maybe_checkpoint
     }
 
+    #[must_use]
     pub fn run(
         random_id: u64,
         max_executing_queue_count: usize,
@@ -1744,7 +1745,7 @@ impl ScheduleStage {
         to_execute_substage: &crossbeam_channel::Sender<ExecutablePayload>,
         from_execute_substage: &crossbeam_channel::Receiver<UnlockablePayload>,
         maybe_to_next_stage: Option<&crossbeam_channel::Sender<ExaminablePayload>>, // assume nonblocking
-    ) {
+    ) -> Option<std::sync::Arc<Checkpoint> {
         #[derive(Clone, Copy, Debug)]
         struct AtTopOfScheduleThread;
         unsafe impl AtScheduleThread for AtTopOfScheduleThread {}

From c30f9e72b6c1c1552a1cdc8e8f2166dd820ce86f Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 15 Sep 2022 22:18:14 +0900
Subject: [PATCH 2643/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 147664b73a68d4..2db9da52379dd3 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1745,7 +1745,7 @@ impl ScheduleStage {
         to_execute_substage: &crossbeam_channel::Sender<ExecutablePayload>,
         from_execute_substage: &crossbeam_channel::Receiver<UnlockablePayload>,
         maybe_to_next_stage: Option<&crossbeam_channel::Sender<ExaminablePayload>>, // assume nonblocking
-    ) -> Option<std::sync::Arc<Checkpoint> {
+    ) -> Option<std::sync::Arc<Checkpoint>> {
         #[derive(Clone, Copy, Debug)]
         struct AtTopOfScheduleThread;
         unsafe impl AtScheduleThread for AtTopOfScheduleThread {}

From 9034a18de421d51f1a5611faec952543e4848950 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 15 Sep 2022 22:18:53 +0900
Subject: [PATCH 2644/3199] save

---
 runtime/src/bank.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 0e27b1606f914b..e3ef53f6d48bb9 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1263,7 +1263,7 @@ impl Scheduler {
         //assert_eq!(index, self.transaction_index.fetch_add(1, std::sync::atomic::Ordering::SeqCst));
         let uw = usize::max_value() - index;
         let t = solana_scheduler::Task::new_for_queue(nast, uw as u64, (sanitized_tx.clone(), locks));
-        self.transaction_sender.as_ref().unwrap().send(solana_scheduler::SchedulablePayload(t)).unwrap();
+        self.transaction_sender.as_ref().unwrap().send(solana_scheduler::SchedulablePayload(solana_scheduler::Flushable::Payload(t))).unwrap();
     }
 }
 

From fcd58f4960b68aafbc7e099b9880dd2a4599f04d Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 15 Sep 2022 22:22:51 +0900
Subject: [PATCH 2645/3199] save

---
 runtime/src/bank.rs | 31 ++++++++++++++++++++-----------
 1 file changed, 20 insertions(+), 11 deletions(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index e3ef53f6d48bb9..ab337fdf99ec52 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1423,22 +1423,31 @@ impl Default for Scheduler {
                 thread_priority::set_current_thread_priority(thread_priority::ThreadPriority::Max).unwrap();
             }
 
-            let mut runnable_queue = solana_scheduler::TaskQueue::default();
             let max_executing_queue_count = std::env::var("MAX_EXECUTING_QUEUE_COUNT")
                 .unwrap_or(format!("{}", 1))
                 .parse::<usize>()
                 .unwrap();
 
-            solana_scheduler::ScheduleStage::run(
-                random_id,
-                max_executing_queue_count,
-                &mut runnable_queue,
-                &mut address_book,
-                &transaction_receiver,
-                &scheduled_ee_sender,
-                &processed_ee_receiver,
-                Some(&retired_ee_sender),
-            );
+            loop {
+                let mut runnable_queue = solana_scheduler::TaskQueue::default();
+                let checkpoint = solana_scheduler::ScheduleStage::run(
+                    random_id,
+                    max_executing_queue_count,
+                    &mut runnable_queue,
+                    &mut address_book,
+                    &transaction_receiver,
+                    &scheduled_ee_sender,
+                    &processed_ee_receiver,
+                    Some(&retired_ee_sender),
+                );
+
+                if let Some(checkpoint) = maybe_checkpoint {
+                    checkpoint.wait_for_restart();
+                } else {
+                    break;
+                }
+            }
+
             drop(transaction_receiver);
             drop(scheduled_ee_sender);
             drop(processed_ee_receiver);

From 7900d73214990153394f29d1b4b861529ca27f10 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 15 Sep 2022 22:23:04 +0900
Subject: [PATCH 2646/3199] save

---
 runtime/src/bank.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index ab337fdf99ec52..aca2ccfc5feef2 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1430,7 +1430,7 @@ impl Default for Scheduler {
 
             loop {
                 let mut runnable_queue = solana_scheduler::TaskQueue::default();
-                let checkpoint = solana_scheduler::ScheduleStage::run(
+                let maybe_checkpoint = solana_scheduler::ScheduleStage::run(
                     random_id,
                     max_executing_queue_count,
                     &mut runnable_queue,

From fe389cdc2d78127ca3beec720072f4ca747bc460 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 15 Sep 2022 22:28:49 +0900
Subject: [PATCH 2647/3199] save

---
 runtime/src/bank.rs | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index aca2ccfc5feef2..7d7bfe322ca528 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1487,7 +1487,15 @@ impl Scheduler {
         //drop(transaction_sender);
         let checkpoint = solana_scheduler::Checkpoint::new(3);
         transaction_sender.send(solana_scheduler::SchedulablePayload(solana_scheduler::Flushable::Flush(checkpoint))).unwrap();
+        checkpoint.wait_for_scheduler();
+        {
+            let mut w = self.scheduler.write().unwrap();
+            *w.bank.write().unwrap() = None;
+            w.slot = None;
+            drop(w);
+        }
 
+        /*
         let executing_thread_duration_pairs: Result<Vec<_>> = self.executing_thread_handles.take().unwrap().into_iter().map(|executing_thread_handle| {
             executing_thread_handle.join().unwrap().map(|u| (u.0.as_micros(), u.1.as_micros()))
         }).collect();
@@ -1504,6 +1512,7 @@ impl Scheduler {
 
         info!("Scheduler::gracefully_stop(): slot: {} id_{:016x} durations 1/2 (cpu ): scheduler: {}us, error_collector: {}us, lanes: {}us = {:?}", self.slot.map(|s| format!("{}", s)).unwrap_or("-".into()), self.random_id, scheduler_thread_cpu_us, error_collector_thread_cpu_us, executing_thread_cpu_us.iter().sum::<u128>(), &executing_thread_cpu_us);
         info!("Scheduler::gracefully_stop(): slot: {} id_{:016x} durations 2/2 (wall): scheduler: {}us, error_collector: {}us, lanes: {}us = {:?}", self.slot.map(|s| format!("{}", s)).unwrap_or("-".into()), self.random_id, scheduler_thread_wall_time_us, error_collector_thread_wall_time_us, executing_thread_wall_time_us.iter().sum::<u128>(), &executing_thread_wall_time_us);
+        */
 
         Ok(())
     }

From 2f2d1a4e83032d01f5a88648fff16642ec1ce7ff Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 15 Sep 2022 22:29:06 +0900
Subject: [PATCH 2648/3199] save

---
 runtime/src/bank.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 7d7bfe322ca528..109a081919c1f1 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1487,7 +1487,7 @@ impl Scheduler {
         //drop(transaction_sender);
         let checkpoint = solana_scheduler::Checkpoint::new(3);
         transaction_sender.send(solana_scheduler::SchedulablePayload(solana_scheduler::Flushable::Flush(checkpoint))).unwrap();
-        checkpoint.wait_for_scheduler();
+        checkpoint.wait_for_restart();
         {
             let mut w = self.scheduler.write().unwrap();
             *w.bank.write().unwrap() = None;

From 4052bf88a32a1137a1cc091f1b1d811dd1230e58 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 15 Sep 2022 22:29:38 +0900
Subject: [PATCH 2649/3199] save

---
 runtime/src/bank.rs | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 109a081919c1f1..ad098b45cb4342 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1489,9 +1489,8 @@ impl Scheduler {
         transaction_sender.send(solana_scheduler::SchedulablePayload(solana_scheduler::Flushable::Flush(checkpoint))).unwrap();
         checkpoint.wait_for_restart();
         {
-            let mut w = self.scheduler.write().unwrap();
-            *w.bank.write().unwrap() = None;
-            w.slot = None;
+            *self.bank.write().unwrap() = None;
+            self.slot = None;
             drop(w);
         }
 

From 12c324d0df031bd3adb3c774a163a1590b89379b Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 15 Sep 2022 22:29:46 +0900
Subject: [PATCH 2650/3199] save

---
 runtime/src/bank.rs | 1 -
 1 file changed, 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index ad098b45cb4342..b8c96cb70595c8 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1491,7 +1491,6 @@ impl Scheduler {
         {
             *self.bank.write().unwrap() = None;
             self.slot = None;
-            drop(w);
         }
 
         /*

From 271212347075e2c1af35e63775ffec2a86b429bc Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 15 Sep 2022 22:30:21 +0900
Subject: [PATCH 2651/3199] save

---
 runtime/src/bank.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index b8c96cb70595c8..80ec0677ba7609 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1486,7 +1486,7 @@ impl Scheduler {
 
         //drop(transaction_sender);
         let checkpoint = solana_scheduler::Checkpoint::new(3);
-        transaction_sender.send(solana_scheduler::SchedulablePayload(solana_scheduler::Flushable::Flush(checkpoint))).unwrap();
+        transaction_sender.send(solana_scheduler::SchedulablePayload(solana_scheduler::Flushable::Flush(std::sync::Arc::clone(checkpoint)))).unwrap();
         checkpoint.wait_for_restart();
         {
             *self.bank.write().unwrap() = None;

From 89d011fe36ba505664530379899f13969257fe18 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 15 Sep 2022 22:30:31 +0900
Subject: [PATCH 2652/3199] save

---
 runtime/src/bank.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 80ec0677ba7609..5c134f8c25d0b7 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1486,7 +1486,7 @@ impl Scheduler {
 
         //drop(transaction_sender);
         let checkpoint = solana_scheduler::Checkpoint::new(3);
-        transaction_sender.send(solana_scheduler::SchedulablePayload(solana_scheduler::Flushable::Flush(std::sync::Arc::clone(checkpoint)))).unwrap();
+        transaction_sender.send(solana_scheduler::SchedulablePayload(solana_scheduler::Flushable::Flush(std::sync::Arc::clone(&checkpoint)))).unwrap();
         checkpoint.wait_for_restart();
         {
             *self.bank.write().unwrap() = None;

From 6a31da47fbe7c907e9cf3d45961cf079beb5a510 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 15 Sep 2022 22:32:24 +0900
Subject: [PATCH 2653/3199] save

---
 runtime/src/bank.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 5c134f8c25d0b7..3089dab1cb4661 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1267,8 +1267,8 @@ impl Scheduler {
     }
 }
 
-impl Default for Scheduler {
-    fn default() -> Self {
+impl Scheduler {
+    fn default2() -> Self {
         let start = Instant::now();
         let mut address_book = solana_scheduler::AddressBook::default();
         let preloader = Arc::new(address_book.preloader());

From 193431356042ee3e1e905435c208f1c13c029110 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 15 Sep 2022 22:32:53 +0900
Subject: [PATCH 2654/3199] save

---
 runtime/src/bank.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 3089dab1cb4661..d17c9279d69dae 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1208,7 +1208,7 @@ impl SchedulerPool {
     }
 
     fn create(&mut self) {
-        self.schedulers.push(Default::default());
+        self.schedulers.push(Scheduler::default2());
     }
 
     fn take_from_pool(&mut self) -> Arc<Scheduler> {

From 847ce7129bedd5604fea6c9fc1cc5f8b7258d441 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 15 Sep 2022 22:33:10 +0900
Subject: [PATCH 2655/3199] save

---
 runtime/src/bank.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index d17c9279d69dae..fbf79addaa7615 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1208,7 +1208,7 @@ impl SchedulerPool {
     }
 
     fn create(&mut self) {
-        self.schedulers.push(Scheduler::default2());
+        self.schedulers.push(Arc::new(Scheduler::default2()));
     }
 
     fn take_from_pool(&mut self) -> Arc<Scheduler> {

From 4405fbb876c73c0e6631d55cf052da6d8fd8121e Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 15 Sep 2022 22:34:58 +0900
Subject: [PATCH 2656/3199] save

---
 runtime/src/bank.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index fbf79addaa7615..711b299849ccbf 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1894,7 +1894,7 @@ impl Bank {
             accounts_data_size_delta_on_chain: AtomicI64::new(0),
             accounts_data_size_delta_off_chain: AtomicI64::new(0),
             fee_structure: FeeStructure::default(),
-            scheduler: Default::default(),
+            scheduler: SCHEDULER_POOL.take_from_pool(),
         };
 
         let accounts_data_size_initial = bank.get_total_accounts_stats().unwrap().data_len as u64;

From 8ccc178730b401948eac5e7d9e6e6eed400e484e Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 15 Sep 2022 22:35:18 +0900
Subject: [PATCH 2657/3199] save

---
 runtime/src/bank.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 711b299849ccbf..c54ec74c11635a 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1894,7 +1894,7 @@ impl Bank {
             accounts_data_size_delta_on_chain: AtomicI64::new(0),
             accounts_data_size_delta_off_chain: AtomicI64::new(0),
             fee_structure: FeeStructure::default(),
-            scheduler: SCHEDULER_POOL.take_from_pool(),
+            scheduler: SCHEDULER_POOL.lock().unwrap().take_from_pool(),
         };
 
         let accounts_data_size_initial = bank.get_total_accounts_stats().unwrap().data_len as u64;

From e06b7a9232bcccb4e6339ad8e0f972fc9fea4623 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 15 Sep 2022 22:35:47 +0900
Subject: [PATCH 2658/3199] save

---
 runtime/src/bank.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index c54ec74c11635a..da6071d65e1aac 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1703,7 +1703,7 @@ pub struct Bank {
 
     pub incremental_snapshot_persistence: Option<BankIncrementalSnapshotPersistence>,
 
-    scheduler: RwLock<Scheduler>,
+    scheduler: RwLock<Arc<Scheduler>>,
 }
 
 struct VoteWithStakeDelegations {

From 56f0770d6be81eac1566f2dfd52436c83a7826c3 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 15 Sep 2022 22:36:19 +0900
Subject: [PATCH 2659/3199] save

---
 runtime/src/bank.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index da6071d65e1aac..0b3c24739afd61 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1894,7 +1894,7 @@ impl Bank {
             accounts_data_size_delta_on_chain: AtomicI64::new(0),
             accounts_data_size_delta_off_chain: AtomicI64::new(0),
             fee_structure: FeeStructure::default(),
-            scheduler: SCHEDULER_POOL.lock().unwrap().take_from_pool(),
+            scheduler: RwLock::new(SCHEDULER_POOL.lock().unwrap().take_from_pool()),
         };
 
         let accounts_data_size_initial = bank.get_total_accounts_stats().unwrap().data_len as u64;

From f793e293117e01fa0f632452f8aca0c9caee9051 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 15 Sep 2022 22:36:46 +0900
Subject: [PATCH 2660/3199] save

---
 runtime/src/bank.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 0b3c24739afd61..7e10363f038559 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1703,7 +1703,7 @@ pub struct Bank {
 
     pub incremental_snapshot_persistence: Option<BankIncrementalSnapshotPersistence>,
 
-    scheduler: RwLock<Arc<Scheduler>>,
+    scheduler: RwLock<Option<Arc<Scheduler>>>,
 }
 
 struct VoteWithStakeDelegations {

From fda375f540cc1cb3d8003d74cc1ebaf897c58ba3 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 15 Sep 2022 22:37:20 +0900
Subject: [PATCH 2661/3199] save

---
 runtime/src/bank.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 7e10363f038559..3a5c3868326abf 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1894,7 +1894,7 @@ impl Bank {
             accounts_data_size_delta_on_chain: AtomicI64::new(0),
             accounts_data_size_delta_off_chain: AtomicI64::new(0),
             fee_structure: FeeStructure::default(),
-            scheduler: RwLock::new(SCHEDULER_POOL.lock().unwrap().take_from_pool()),
+            scheduler: RwLock::new(Some(SCHEDULER_POOL.lock().unwrap().take_from_pool())),
         };
 
         let accounts_data_size_initial = bank.get_total_accounts_stats().unwrap().data_len as u64;

From f732469f0eb84fe8e49a3906e68b79a9fc59795c Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 15 Sep 2022 22:38:46 +0900
Subject: [PATCH 2662/3199] save

---
 runtime/src/bank.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 3a5c3868326abf..8225d9f57d375a 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -3760,7 +3760,7 @@ impl Bank {
         // committed before this write lock can be obtained here.
         let mut hash = self.hash.write().unwrap();
         if *hash == Hash::default() {
-            assert!(self.scheduler.read().unwrap().graceful_stop_initiated);
+            assert!(self.scheduler.read().unwrap().unwrap().graceful_stop_initiated);
             // finish up any deferred changes to account state
             self.collect_rent_eagerly(false);
             self.collect_fees();

From 51f20741010a9f3b16b5496b3b329b488a9f73fb Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 15 Sep 2022 22:39:03 +0900
Subject: [PATCH 2663/3199] save

---
 runtime/src/bank.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 8225d9f57d375a..9cf8f28b4cb8ab 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -6560,7 +6560,7 @@ impl Bank {
         assert_eq!(bank.slot(), batch.bank().slot());
 
         let scheduler = {
-            let r = self.scheduler.read().unwrap();
+            let r = self.scheduler.read().unwrap().unwrap();
 
             if r.bank.read().unwrap().is_none() {
                 drop(r);

From c412697dbf425dccde968b26f721b071bfeea13d Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 15 Sep 2022 22:39:17 +0900
Subject: [PATCH 2664/3199] save

---
 runtime/src/bank.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 9cf8f28b4cb8ab..37537ec3d36f21 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -6566,7 +6566,7 @@ impl Bank {
                 drop(r);
                 // this relocking (r=>w) is racy here; we want parking_lot's upgrade; but overwriting should be
                 // safe.
-                let mut w = self.scheduler.write().unwrap();
+                let mut w = self.scheduler.write().unwrap().unwrap();
                 *w.bank.write().unwrap() = Some(Arc::downgrade(&bank));
                 w.slot = Some(bank.slot);
                 drop(w);

From f8d6c58716da2aa04e60d1281cf477c1299f8b1a Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 15 Sep 2022 22:39:40 +0900
Subject: [PATCH 2665/3199] save

---
 runtime/src/bank.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 37537ec3d36f21..92204b4d2b78f8 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -6570,7 +6570,7 @@ impl Bank {
                 *w.bank.write().unwrap() = Some(Arc::downgrade(&bank));
                 w.slot = Some(bank.slot);
                 drop(w);
-                self.scheduler.read().unwrap()
+                self.scheduler.read().unwrap().unwrap()
             } else {
                 assert_eq!(bank.slot(), r.slot.unwrap());
                 r

From cdb2d50055db25b1dd8230a6656e059a4c9a0877 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 15 Sep 2022 22:40:00 +0900
Subject: [PATCH 2666/3199] save

---
 runtime/src/bank.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 92204b4d2b78f8..c830933ed7de50 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -6585,7 +6585,7 @@ impl Bank {
     pub fn handle_aborted_transactions(
         &self,
     ) -> Vec<Result<()>> {
-        let scheduler = self.scheduler.read().unwrap();
+        let scheduler = self.scheduler.read().unwrap().unwrap();
         scheduler.handle_aborted_executions()
     }
 

From ebe7744ef6f76c4bb00f50f81f2402f5213f5640 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 15 Sep 2022 22:40:16 +0900
Subject: [PATCH 2667/3199] save

---
 runtime/src/bank.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index c830933ed7de50..a111f68aee677f 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -8238,7 +8238,7 @@ impl Bank {
     }
 
     pub fn wait_for_scheduler(&self) -> Result<()> {
-        let mut scheduler = self.scheduler.write().unwrap();
+        let mut scheduler = self.scheduler.write().unwrap().unwrap();
         scheduler.gracefully_stop()?;
         scheduler.handle_aborted_executions().into_iter().next().unwrap_or(Ok(()))
     }

From 5f72f71d3e7af820f371642dcf1dd595175ccb8a Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 15 Sep 2022 22:43:26 +0900
Subject: [PATCH 2668/3199] save

---
 runtime/src/bank.rs | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index a111f68aee677f..95f7be60738021 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -4197,12 +4197,13 @@ impl Bank {
         // readers can starve this write lock acquisition and ticks would be slowed down too
         // much if the write lock is acquired for each tick.
         let mut w_blockhash_queue = self.blockhash_queue.write().unwrap();
-        let new_scheduler = Scheduler::default();
+        //let new_scheduler = Scheduler::default();
         if maybe_last_error.is_err() {
             warn!("register_recent_blockhash: carrying over this error: {:?}", maybe_last_error);
-            new_scheduler.collected_errors.lock().unwrap().push(maybe_last_error);
+            //new_scheduler.collected_errors.lock().unwrap().push(maybe_last_error);
+            *self.scheduler.write().unwrap().unwrap().collected_errors.lock().unwrap().push(maybe_last_error);
         }
-        *self.scheduler.write().unwrap() = new_scheduler;
+        //*self.scheduler.write().unwrap() = new_scheduler;
 
         info!("register_recent_blockhash: slot: {} reinitializing the scheduler: end", self.slot());
 

From 750e469139be59eef968c2f7c83dbc30caef08e6 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 15 Sep 2022 22:43:42 +0900
Subject: [PATCH 2669/3199] save

---
 runtime/src/bank.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 95f7be60738021..9fe6f8bcbf0bed 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -4201,7 +4201,7 @@ impl Bank {
         if maybe_last_error.is_err() {
             warn!("register_recent_blockhash: carrying over this error: {:?}", maybe_last_error);
             //new_scheduler.collected_errors.lock().unwrap().push(maybe_last_error);
-            *self.scheduler.write().unwrap().unwrap().collected_errors.lock().unwrap().push(maybe_last_error);
+            self.scheduler.write().unwrap().unwrap().collected_errors.lock().unwrap().push(maybe_last_error);
         }
         //*self.scheduler.write().unwrap() = new_scheduler;
 

From fc67c4b46e43e870b71a47654b3952a4037680ae Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 15 Sep 2022 22:44:05 +0900
Subject: [PATCH 2670/3199] save

---
 runtime/src/bank.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 9fe6f8bcbf0bed..938d565b7003c1 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -8239,7 +8239,7 @@ impl Bank {
     }
 
     pub fn wait_for_scheduler(&self) -> Result<()> {
-        let mut scheduler = self.scheduler.write().unwrap().unwrap();
+        let mut scheduler = self.scheduler.write().unwrap().as_ref().unwrap();
         scheduler.gracefully_stop()?;
         scheduler.handle_aborted_executions().into_iter().next().unwrap_or(Ok(()))
     }

From 32fe31986afef41d9608e74c97be2516ffcc817c Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 15 Sep 2022 22:44:36 +0900
Subject: [PATCH 2671/3199] save

---
 runtime/src/bank.rs | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 938d565b7003c1..a41a031b1b3cf1 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -8239,7 +8239,8 @@ impl Bank {
     }
 
     pub fn wait_for_scheduler(&self) -> Result<()> {
-        let mut scheduler = self.scheduler.write().unwrap().as_ref().unwrap();
+        let s = self.scheduler.write().unwrap();
+        let mut scheduler = s.unwrap();
         scheduler.gracefully_stop()?;
         scheduler.handle_aborted_executions().into_iter().next().unwrap_or(Ok(()))
     }

From e69ab99460736871b7e94b8b7655dbaab90249e2 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 15 Sep 2022 22:44:50 +0900
Subject: [PATCH 2672/3199] save

---
 runtime/src/bank.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index a41a031b1b3cf1..a6160ec50bc905 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -8240,7 +8240,7 @@ impl Bank {
 
     pub fn wait_for_scheduler(&self) -> Result<()> {
         let s = self.scheduler.write().unwrap();
-        let mut scheduler = s.unwrap();
+        let mut scheduler = s.as_ref().unwrap();
         scheduler.gracefully_stop()?;
         scheduler.handle_aborted_executions().into_iter().next().unwrap_or(Ok(()))
     }

From 911c3c7c374fe4bd32e0058e9371d5a6dcb26e34 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 15 Sep 2022 22:46:37 +0900
Subject: [PATCH 2673/3199] save

---
 runtime/src/bank.rs | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index a6160ec50bc905..53d364851166a9 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1474,19 +1474,21 @@ impl Scheduler {
 }
 
 impl Scheduler {
-    fn gracefully_stop(&mut self) -> Result<()> {
+    fn gracefully_stop(&self) -> Result<()> {
+        /*
         if self.graceful_stop_initiated {
             info!("Scheduler::gracefully_stop(): id_{:016x} (skipped..?)", self.random_id);
             return Ok(());
         }
         self.graceful_stop_initiated = true;
+        */
 
         info!("Scheduler::gracefully_stop(): id_{:016x} waiting..", self.random_id);
-        let transaction_sender = self.transaction_sender.take().unwrap();
+        //let transaction_sender = self.transaction_sender.take().unwrap();
 
         //drop(transaction_sender);
         let checkpoint = solana_scheduler::Checkpoint::new(3);
-        transaction_sender.send(solana_scheduler::SchedulablePayload(solana_scheduler::Flushable::Flush(std::sync::Arc::clone(&checkpoint)))).unwrap();
+        self.transaction_sender.unwrap().send(solana_scheduler::SchedulablePayload(solana_scheduler::Flushable::Flush(std::sync::Arc::clone(&checkpoint)))).unwrap();
         checkpoint.wait_for_restart();
         {
             *self.bank.write().unwrap() = None;

From 78cb8c902205147e44ccd2613efb48dfece85860 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 15 Sep 2022 22:47:05 +0900
Subject: [PATCH 2674/3199] save

---
 runtime/src/bank.rs | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 53d364851166a9..c463222cdd7adb 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -6588,7 +6588,8 @@ impl Bank {
     pub fn handle_aborted_transactions(
         &self,
     ) -> Vec<Result<()>> {
-        let scheduler = self.scheduler.read().unwrap().unwrap();
+        let s = self.scheduler.read().unwrap();
+        let scheduler = s.as_ref().unwrap();
         scheduler.handle_aborted_executions()
     }
 

From ff141ab3ae4cbaa330b766d9c0787ed67fcf7cb9 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 15 Sep 2022 22:48:47 +0900
Subject: [PATCH 2675/3199] save

---
 runtime/src/bank.rs | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index c463222cdd7adb..503264df0859da 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -6562,10 +6562,10 @@ impl Bank {
     ) {
         assert_eq!(bank.slot(), batch.bank().slot());
 
-        let scheduler = {
-            let r = self.scheduler.read().unwrap().unwrap();
+        let s = {
+            let r = self.scheduler.read().unwrap()
 
-            if r.bank.read().unwrap().is_none() {
+            if r.as_ref().unwrap().bank.read().unwrap().is_none() {
                 drop(r);
                 // this relocking (r=>w) is racy here; we want parking_lot's upgrade; but overwriting should be
                 // safe.
@@ -6573,12 +6573,13 @@ impl Bank {
                 *w.bank.write().unwrap() = Some(Arc::downgrade(&bank));
                 w.slot = Some(bank.slot);
                 drop(w);
-                self.scheduler.read().unwrap().unwrap()
+                self.scheduler.read().unwrap()
             } else {
-                assert_eq!(bank.slot(), r.slot.unwrap());
+                assert_eq!(bank.slot(), r.as_ref().unwrap().slot.unwrap());
                 r
             }
         };
+        let scheduler = s.as_ref().unwrap();
 
         for (st, &i) in batch.sanitized_transactions().iter().zip(transaction_indexes.iter()) {
             scheduler.schedule(st, i);

From f3ebba19d2b5a68c74e9ec896c4029ab9702b826 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 15 Sep 2022 22:48:58 +0900
Subject: [PATCH 2676/3199] save

---
 runtime/src/bank.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 503264df0859da..4501563a2349b2 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -6563,7 +6563,7 @@ impl Bank {
         assert_eq!(bank.slot(), batch.bank().slot());
 
         let s = {
-            let r = self.scheduler.read().unwrap()
+            let r = self.scheduler.read().unwrap();
 
             if r.as_ref().unwrap().bank.read().unwrap().is_none() {
                 drop(r);

From 0aa86a9fe53c6d57ac1571ff7b04106792d6f01c Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 15 Sep 2022 22:49:57 +0900
Subject: [PATCH 2677/3199] save

---
 runtime/src/bank.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 4501563a2349b2..ace9d28f5aa21a 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1239,7 +1239,7 @@ struct Scheduler {
     graceful_stop_initiated: bool,
     collected_errors: Arc<std::sync::Mutex<Vec<Result<()>>>>,
     bank: std::sync::Arc<std::sync::RwLock<std::option::Option<std::sync::Weak<Bank>>>>,
-    slot: Option<Slot>,
+    slot: Option<AtomicU64>,
 }
 
 impl Scheduler {

From fece606578d4ff9873846a2b3318d709528cf948 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 15 Sep 2022 22:50:24 +0900
Subject: [PATCH 2678/3199] save

---
 runtime/src/bank.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index ace9d28f5aa21a..d5dc772fc2dae0 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -6571,7 +6571,7 @@ impl Bank {
                 // safe.
                 let mut w = self.scheduler.write().unwrap().unwrap();
                 *w.bank.write().unwrap() = Some(Arc::downgrade(&bank));
-                w.slot = Some(bank.slot);
+                w.slot = Some(AtomicU64::new(bank.slot));
                 drop(w);
                 self.scheduler.read().unwrap()
             } else {

From 80d1baa5235cf56cc6da501618ab022c85785a92 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 15 Sep 2022 22:51:00 +0900
Subject: [PATCH 2679/3199] save

---
 runtime/src/bank.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index d5dc772fc2dae0..32dda691da7a28 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -6575,7 +6575,7 @@ impl Bank {
                 drop(w);
                 self.scheduler.read().unwrap()
             } else {
-                assert_eq!(bank.slot(), r.as_ref().unwrap().slot.unwrap());
+                assert_eq!(bank.slot(), r.as_ref().unwrap().slot.unwrap().load(SeqCst));
                 r
             }
         };

From 218b2180de52205dbd6cdb8f65c32ecb66ac7faf Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 15 Sep 2022 22:51:15 +0900
Subject: [PATCH 2680/3199] save

---
 runtime/src/bank.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 32dda691da7a28..04736f6b90519b 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -6575,7 +6575,7 @@ impl Bank {
                 drop(w);
                 self.scheduler.read().unwrap()
             } else {
-                assert_eq!(bank.slot(), r.as_ref().unwrap().slot.unwrap().load(SeqCst));
+                assert_eq!(bank.slot(), r.as_ref().unwrap().slot.unwrap().load(std::sync::atomic::Ordering::SeqCst));
                 r
             }
         };

From 1257094b5801506010d1c3c62f5e7b269f7ffddb Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 15 Sep 2022 22:51:58 +0900
Subject: [PATCH 2681/3199] save

---
 runtime/src/bank.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 04736f6b90519b..0754c68f3205b6 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -6571,7 +6571,7 @@ impl Bank {
                 // safe.
                 let mut w = self.scheduler.write().unwrap().unwrap();
                 *w.bank.write().unwrap() = Some(Arc::downgrade(&bank));
-                w.slot = Some(AtomicU64::new(bank.slot));
+                w.slot.unwrap.store(bank.slot(), std::sync::atomic::Ordering::SeqCst);
                 drop(w);
                 self.scheduler.read().unwrap()
             } else {

From 01faca0f23fac971e389599d97468c2306ecf360 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 15 Sep 2022 22:52:50 +0900
Subject: [PATCH 2682/3199] save

---
 runtime/src/bank.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 0754c68f3205b6..a03512eae005b1 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -6571,7 +6571,7 @@ impl Bank {
                 // safe.
                 let mut w = self.scheduler.write().unwrap().unwrap();
                 *w.bank.write().unwrap() = Some(Arc::downgrade(&bank));
-                w.slot.unwrap.store(bank.slot(), std::sync::atomic::Ordering::SeqCst);
+                w.slot.unwrap().store(bank.slot(), std::sync::atomic::Ordering::SeqCst);
                 drop(w);
                 self.scheduler.read().unwrap()
             } else {

From d62beacb4cf3c4a0e189e66404548c67b61ff234 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 15 Sep 2022 22:53:21 +0900
Subject: [PATCH 2683/3199] save

---
 runtime/src/bank.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index a03512eae005b1..7eb645a74b3bbc 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -6575,7 +6575,7 @@ impl Bank {
                 drop(w);
                 self.scheduler.read().unwrap()
             } else {
-                assert_eq!(bank.slot(), r.as_ref().unwrap().slot.unwrap().load(std::sync::atomic::Ordering::SeqCst));
+                assert_eq!(bank.slot(), r.as_ref().unwrap().slot.as_ref().unwrap().load(std::sync::atomic::Ordering::SeqCst));
                 r
             }
         };

From 6b767e5c83834def2caba35831a389277ead143e Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 15 Sep 2022 22:53:40 +0900
Subject: [PATCH 2684/3199] save

---
 runtime/src/bank.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 7eb645a74b3bbc..31315c6c16b7b2 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -6571,7 +6571,7 @@ impl Bank {
                 // safe.
                 let mut w = self.scheduler.write().unwrap().unwrap();
                 *w.bank.write().unwrap() = Some(Arc::downgrade(&bank));
-                w.slot.unwrap().store(bank.slot(), std::sync::atomic::Ordering::SeqCst);
+                w.slot.as_ref().unwrap().store(bank.slot(), std::sync::atomic::Ordering::SeqCst);
                 drop(w);
                 self.scheduler.read().unwrap()
             } else {

From 86fa706bd765ef336ee800095a7cfac67d75fe49 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 15 Sep 2022 22:54:11 +0900
Subject: [PATCH 2685/3199] save

---
 runtime/src/bank.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 31315c6c16b7b2..f41b09377f4cb8 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -6569,7 +6569,7 @@ impl Bank {
                 drop(r);
                 // this relocking (r=>w) is racy here; we want parking_lot's upgrade; but overwriting should be
                 // safe.
-                let mut w = self.scheduler.write().unwrap().unwrap();
+                let mut w = self.scheduler.write().unwrap().as_ref().unwrap();
                 *w.bank.write().unwrap() = Some(Arc::downgrade(&bank));
                 w.slot.as_ref().unwrap().store(bank.slot(), std::sync::atomic::Ordering::SeqCst);
                 drop(w);

From 52001e9d105b351327e8063e46f55e5804ce44ab Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 15 Sep 2022 22:54:37 +0900
Subject: [PATCH 2686/3199] save

---
 runtime/src/bank.rs | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index f41b09377f4cb8..48d076546165aa 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -6569,7 +6569,8 @@ impl Bank {
                 drop(r);
                 // this relocking (r=>w) is racy here; we want parking_lot's upgrade; but overwriting should be
                 // safe.
-                let mut w = self.scheduler.write().unwrap().as_ref().unwrap();
+                let ss = self.scheduler.write().unwrap();
+                let mut w = ss.as_ref().unwrap();
                 *w.bank.write().unwrap() = Some(Arc::downgrade(&bank));
                 w.slot.as_ref().unwrap().store(bank.slot(), std::sync::atomic::Ordering::SeqCst);
                 drop(w);

From 8b64c1439e988095eeae087a7faf2d31b360b012 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 15 Sep 2022 22:55:21 +0900
Subject: [PATCH 2687/3199] save

---
 runtime/src/bank.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 48d076546165aa..e2b4b6d90eb228 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -4203,7 +4203,7 @@ impl Bank {
         if maybe_last_error.is_err() {
             warn!("register_recent_blockhash: carrying over this error: {:?}", maybe_last_error);
             //new_scheduler.collected_errors.lock().unwrap().push(maybe_last_error);
-            self.scheduler.write().unwrap().unwrap().collected_errors.lock().unwrap().push(maybe_last_error);
+            self.scheduler.write().unwrap().as_ref().unwrap().collected_errors.lock().unwrap().push(maybe_last_error);
         }
         //*self.scheduler.write().unwrap() = new_scheduler;
 

From 06e376df7b730c0bb83ec68a67f10e7a0cfef30c Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 15 Sep 2022 22:55:41 +0900
Subject: [PATCH 2688/3199] save

---
 runtime/src/bank.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index e2b4b6d90eb228..958a34a384f0de 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -3762,7 +3762,7 @@ impl Bank {
         // committed before this write lock can be obtained here.
         let mut hash = self.hash.write().unwrap();
         if *hash == Hash::default() {
-            assert!(self.scheduler.read().unwrap().unwrap().graceful_stop_initiated);
+            assert!(self.scheduler.read().unwrap().as_ref().unwrap().graceful_stop_initiated);
             // finish up any deferred changes to account state
             self.collect_rent_eagerly(false);
             self.collect_fees();

From fd74ed62704a01d3da20b01dad053267b91a32ad Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 15 Sep 2022 22:56:52 +0900
Subject: [PATCH 2689/3199] save

---
 runtime/src/bank.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 958a34a384f0de..142f2c8db5458a 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1492,7 +1492,7 @@ impl Scheduler {
         checkpoint.wait_for_restart();
         {
             *self.bank.write().unwrap() = None;
-            self.slot = None;
+            self.slot.unwrap().store(0, std::sync::atomic::Ordering::SeqCst);
         }
 
         /*

From 923cc61dc21bad94eea171c1af586b9e162bf6a7 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 15 Sep 2022 22:57:12 +0900
Subject: [PATCH 2690/3199] save

---
 runtime/src/bank.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 142f2c8db5458a..10fc8401002b52 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1492,7 +1492,7 @@ impl Scheduler {
         checkpoint.wait_for_restart();
         {
             *self.bank.write().unwrap() = None;
-            self.slot.unwrap().store(0, std::sync::atomic::Ordering::SeqCst);
+            self.slot.as_ref().unwrap().store(0, std::sync::atomic::Ordering::SeqCst);
         }
 
         /*

From 91e1599f634e01dbc54b66f51f71e3b018afead1 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 15 Sep 2022 22:57:33 +0900
Subject: [PATCH 2691/3199] save

---
 runtime/src/bank.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 10fc8401002b52..9031998b4ea6aa 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1488,7 +1488,7 @@ impl Scheduler {
 
         //drop(transaction_sender);
         let checkpoint = solana_scheduler::Checkpoint::new(3);
-        self.transaction_sender.unwrap().send(solana_scheduler::SchedulablePayload(solana_scheduler::Flushable::Flush(std::sync::Arc::clone(&checkpoint)))).unwrap();
+        self.transaction_sender.as_ref().unwrap().send(solana_scheduler::SchedulablePayload(solana_scheduler::Flushable::Flush(std::sync::Arc::clone(&checkpoint)))).unwrap();
         checkpoint.wait_for_restart();
         {
             *self.bank.write().unwrap() = None;

From 081244086e5aff01d877ead409a2a5bfc6eeb076 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 15 Sep 2022 23:00:13 +0900
Subject: [PATCH 2692/3199] save

---
 runtime/src/bank.rs | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 9031998b4ea6aa..7ec4bbc3ea3d15 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -8244,10 +8244,11 @@ impl Bank {
     }
 
     pub fn wait_for_scheduler(&self) -> Result<()> {
-        let s = self.scheduler.write().unwrap();
+        let s = self.scheduler.write().take();
         let mut scheduler = s.as_ref().unwrap();
         scheduler.gracefully_stop()?;
-        scheduler.handle_aborted_executions().into_iter().next().unwrap_or(Ok(()))
+        let e = scheduler.handle_aborted_executions().into_iter().next().unwrap_or(Ok(()));
+        e
     }
 }
 

From d341ad2e7b2e3a600db59a7f67897caff863e94d Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 15 Sep 2022 23:00:28 +0900
Subject: [PATCH 2693/3199] save

---
 runtime/src/bank.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 7ec4bbc3ea3d15..51bf2f3d7d8fd7 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -8244,7 +8244,7 @@ impl Bank {
     }
 
     pub fn wait_for_scheduler(&self) -> Result<()> {
-        let s = self.scheduler.write().take();
+        let s = self.scheduler.write().unwrap().take();
         let mut scheduler = s.as_ref().unwrap();
         scheduler.gracefully_stop()?;
         let e = scheduler.handle_aborted_executions().into_iter().next().unwrap_or(Ok(()));

From 117275e2bdb77fab607cd23b5e6940711490f64b Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 15 Sep 2022 23:01:06 +0900
Subject: [PATCH 2694/3199] save

---
 runtime/src/bank.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 51bf2f3d7d8fd7..7ae3c312dac20a 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -8244,7 +8244,7 @@ impl Bank {
     }
 
     pub fn wait_for_scheduler(&self) -> Result<()> {
-        let s = self.scheduler.write().unwrap().take();
+        let s: usize = self.scheduler.write().unwrap().take();
         let mut scheduler = s.as_ref().unwrap();
         scheduler.gracefully_stop()?;
         let e = scheduler.handle_aborted_executions().into_iter().next().unwrap_or(Ok(()));

From 51139a967fc92475b58033ed4032ebbbf5a3919c Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 15 Sep 2022 23:01:21 +0900
Subject: [PATCH 2695/3199] save

---
 runtime/src/bank.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 7ae3c312dac20a..20a9dcd03b9d09 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -8244,7 +8244,7 @@ impl Bank {
     }
 
     pub fn wait_for_scheduler(&self) -> Result<()> {
-        let s: usize = self.scheduler.write().unwrap().take();
+        let s: usize = self.scheduler.write().unwrap().take().unwrap();
         let mut scheduler = s.as_ref().unwrap();
         scheduler.gracefully_stop()?;
         let e = scheduler.handle_aborted_executions().into_iter().next().unwrap_or(Ok(()));

From 6033a99ff8410f87720e3a56e26123dfe1d9b0b7 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 15 Sep 2022 23:02:26 +0900
Subject: [PATCH 2696/3199] save

---
 runtime/src/bank.rs | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 20a9dcd03b9d09..54f7c7b1b94f17 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -8244,10 +8244,10 @@ impl Bank {
     }
 
     pub fn wait_for_scheduler(&self) -> Result<()> {
-        let s: usize = self.scheduler.write().unwrap().take().unwrap();
-        let mut scheduler = s.as_ref().unwrap();
-        scheduler.gracefully_stop()?;
+        let scheduler = self.scheduler.write().unwrap().take().unwrap();
+        scheduler.gracefully_stop().unwrap();
         let e = scheduler.handle_aborted_executions().into_iter().next().unwrap_or(Ok(()));
+        SCHEDULER_POOL.return_to_pool(scheduler);
         e
     }
 }

From 070be0bf1ecec8048453857bd6127646137845c6 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 15 Sep 2022 23:02:40 +0900
Subject: [PATCH 2697/3199] save

---
 runtime/src/bank.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 54f7c7b1b94f17..8705207eaa820d 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -8247,7 +8247,7 @@ impl Bank {
         let scheduler = self.scheduler.write().unwrap().take().unwrap();
         scheduler.gracefully_stop().unwrap();
         let e = scheduler.handle_aborted_executions().into_iter().next().unwrap_or(Ok(()));
-        SCHEDULER_POOL.return_to_pool(scheduler);
+        SCHEDULER_POOL.lock().unwrap().return_to_pool(scheduler);
         e
     }
 }

From 480369f0519894e4f93cee345710f0310fbaf122 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 15 Sep 2022 23:03:24 +0900
Subject: [PATCH 2698/3199] save

---
 runtime/src/bank.rs | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 8705207eaa820d..438e34e9853446 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1524,9 +1524,10 @@ impl Scheduler {
 
 impl Drop for Scheduler {
     fn drop(&mut self) {
-        info!("Scheduler::drop(): id_{:016x} begin..", self.random_id);
-        self.gracefully_stop().unwrap();
-        info!("Scheduler::drop(): id_{:016x} end...", self.random_id);
+        todo!();
+        //info!("Scheduler::drop(): id_{:016x} begin..", self.random_id);
+        //self.gracefully_stop().unwrap();
+        //info!("Scheduler::drop(): id_{:016x} end...", self.random_id);
     }
 }
 

From 55fefebdbab631b214c5ef03303f5f9f94e15df3 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 15 Sep 2022 23:05:31 +0900
Subject: [PATCH 2699/3199] save

---
 runtime/src/bank.rs | 1 +
 1 file changed, 1 insertion(+)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 438e34e9853446..019eba27007f35 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1222,6 +1222,7 @@ impl SchedulerPool {
 
     fn return_to_pool(&mut self, scheduler: Arc<Scheduler>) {
         assert_eq!(1, Arc::strong_count(&scheduler));
+        assert!(scheduler.collected_errors.lock().unwrap().is_empty());
         self.schedulers.push(scheduler);
     }
 }

From 45fd2e7b76c487072e8deca86033bf61795a3519 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 16 Sep 2022 14:06:51 +0900
Subject: [PATCH 2700/3199] save

---
 runtime/src/bank.rs | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 019eba27007f35..f2895e1ed3dded 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1213,6 +1213,7 @@ impl SchedulerPool {
 
     fn take_from_pool(&mut self) -> Arc<Scheduler> {
         if let Some(scheduler) = self.schedulers.pop() {
+            info!("SchedulerPoom: {} is taken...", scheduler.random_id);
             scheduler
         } else {
             self.create();
@@ -1221,6 +1222,7 @@ impl SchedulerPool {
     }
 
     fn return_to_pool(&mut self, scheduler: Arc<Scheduler>) {
+        info!("SchedulerPool: {} is returned...", scheduler.random_id);
         assert_eq!(1, Arc::strong_count(&scheduler));
         assert!(scheduler.collected_errors.lock().unwrap().is_empty());
         self.schedulers.push(scheduler);

From e0e76750e55573e34c3a1cb571726d83e13829c0 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 16 Sep 2022 14:07:44 +0900
Subject: [PATCH 2701/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 2db9da52379dd3..9c1beed41e9a6d 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1768,7 +1768,7 @@ impl ScheduleStage {
 pub struct SchedulablePayload(pub Flushable<TaskInQueue>);
 pub struct ExecutablePayload(pub Box<ExecutionEnvironment>);
 pub struct UnlockablePayload(pub Box<ExecutionEnvironment>);
-pub struct ExaminablePayload(pub Flushable<Box<ExecutionEnvironment>>);
+pub struct ExaminablePayload(pub Box<ExecutionEnvironment>);
 
 pub struct Checkpoint(pub std::sync::Mutex<usize>, pub std::sync::Condvar);
 

From d0e7fd9f180834f2f27622fb7a2f0a46fb6e6085 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 16 Sep 2022 14:08:13 +0900
Subject: [PATCH 2702/3199] save

---
 scheduler/src/lib.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 9c1beed41e9a6d..48c39e2da6f6be 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1525,7 +1525,7 @@ impl ScheduleStage {
                            executing_queue_count = executing_queue_count.checked_sub(1).unwrap();
                            processed_count = processed_count.checked_add(1).unwrap();
                            Self::commit_processed_execution(ast, &mut processed_execution_environment, address_book, &mut commit_clock, &mut provisioning_tracker_count);
-                           to_next_stage.send(ExaminablePayload(Flushable::Payload(processed_execution_environment))).unwrap();
+                           to_next_stage.send(ExaminablePayload(processed_execution_environment)).unwrap();
                        } else {
                            assert_eq!(from_exec.len(), 0);
                            from_exec_disconnected = true;
@@ -1692,7 +1692,7 @@ impl ScheduleStage {
                             &mut commit_clock,
                             &mut provisioning_tracker_count,
                         );
-                        to_next_stage.send(ExaminablePayload(Flushable::Payload(processed_execution_environment))).unwrap();
+                        to_next_stage.send(ExaminablePayload(processed_execution_environment)).unwrap();
                     }
                     if !empty_from {
                         let SchedulablePayload(schedulable) = from_prev.recv().unwrap();

From 996ab82daf83e9c4e7ed97db32a5756b207ecee8 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 16 Sep 2022 14:08:38 +0900
Subject: [PATCH 2703/3199] save

---
 scheduler/src/lib.rs | 8 --------
 1 file changed, 8 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 48c39e2da6f6be..85501da88d4e23 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1460,19 +1460,11 @@ impl ScheduleStage {
                     let nast = NotAtTopOfScheduleThread;
 
                     while let Ok(ExaminablePayload(a)) = ee_receiver.recv() {
-                        match a {
-                            Flushable::Flush(checkpoint) => {
-                                checkpoint.wait_for_restart();
-                                continue;
-                            },
-                            Flushable::Payload(a) => {
                                 assert!(a.task.lock_attempts_not_mut(nast).is_empty());
                                 //assert!(a.task.sequence_time() != usize::max_value());
                                 //let lock_attempts = std::mem::take(&mut a.lock_attempts);
                                 //drop(lock_attempts);
                                 //TaskInQueue::get_mut(&mut a.task).unwrap();
-                            },
-                        }
                     }
                     assert_eq!(ee_receiver.len(), 0);
                     Ok::<(), ()>(())

From f7661327a3061da7110c120f5d313bb11b2a4f06 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 16 Sep 2022 14:09:03 +0900
Subject: [PATCH 2704/3199] save

---
 runtime/src/bank.rs | 8 +-------
 1 file changed, 1 insertion(+), 7 deletions(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index f2895e1ed3dded..ff767701d9a84b 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1396,12 +1396,7 @@ impl Scheduler {
                 thread_priority::set_current_thread_priority(thread_priority::ThreadPriority::Max).unwrap();
             }
 
-            while let Ok(solana_scheduler::ExaminablePayload(a)) = retired_ee_receiver.recv() {
-                match a {
-                    solana_scheduler::Flushable::Flush(checkpoint) => {
-                        checkpoint.wait_for_restart();
-                        continue;
-                    },
+            while let Ok(solana_scheduler::ExaminablePayload(mut ee)) = retired_ee_receiver.recv() {
                     solana_scheduler::Flushable::Payload(mut ee) => {
                         if ee.is_aborted() {
                             warn!(
@@ -1412,7 +1407,6 @@ impl Scheduler {
                         }
                         drop(ee);
                     },
-                }
             }
 
             Ok((started.0.elapsed(), started.1.elapsed()))

From 91a500c4d5a4306fe0ff919da4d81117ea34b9fc Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 16 Sep 2022 14:09:26 +0900
Subject: [PATCH 2705/3199] save

---
 runtime/src/bank.rs | 2 --
 1 file changed, 2 deletions(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index ff767701d9a84b..6bc85b3b1020f7 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1397,7 +1397,6 @@ impl Scheduler {
             }
 
             while let Ok(solana_scheduler::ExaminablePayload(mut ee)) = retired_ee_receiver.recv() {
-                    solana_scheduler::Flushable::Payload(mut ee) => {
                         if ee.is_aborted() {
                             warn!(
                                 "scheduler: Unexpected validator error: {:?}, transaction: {:?}",
@@ -1406,7 +1405,6 @@ impl Scheduler {
                             collected_errors_in_collector_thread.lock().unwrap().push(ee.execution_result.take().unwrap());
                         }
                         drop(ee);
-                    },
             }
 
             Ok((started.0.elapsed(), started.1.elapsed()))

From 53082a5dbdd2d65dcd1df9795554898e31591160 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 16 Sep 2022 14:13:00 +0900
Subject: [PATCH 2706/3199] save

---
 runtime/src/bank.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 6bc85b3b1020f7..0ce314301ce72f 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1242,7 +1242,7 @@ struct Scheduler {
     graceful_stop_initiated: bool,
     collected_errors: Arc<std::sync::Mutex<Vec<Result<()>>>>,
     bank: std::sync::Arc<std::sync::RwLock<std::option::Option<std::sync::Weak<Bank>>>>,
-    slot: Option<AtomicU64>,
+    slot: AtomicU64,
 }
 
 impl Scheduler {

From 8d369d8107fb33c8b0e2fc7ffc9e93c57a25beb6 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 16 Sep 2022 14:13:27 +0900
Subject: [PATCH 2707/3199] save

---
 runtime/src/bank.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 0ce314301ce72f..05c1c708bd5565 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1487,7 +1487,7 @@ impl Scheduler {
         checkpoint.wait_for_restart();
         {
             *self.bank.write().unwrap() = None;
-            self.slot.as_ref().unwrap().store(0, std::sync::atomic::Ordering::SeqCst);
+            self.slot.store(0, std::sync::atomic::Ordering::SeqCst);
         }
 
         /*

From 0c33dc3ee4ee31ad819fde079d7cc9773cc30563 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 16 Sep 2022 14:13:44 +0900
Subject: [PATCH 2708/3199] save

---
 runtime/src/bank.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 05c1c708bd5565..305a25dcbaea45 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -6568,7 +6568,7 @@ impl Bank {
                 let ss = self.scheduler.write().unwrap();
                 let mut w = ss.as_ref().unwrap();
                 *w.bank.write().unwrap() = Some(Arc::downgrade(&bank));
-                w.slot.as_ref().unwrap().store(bank.slot(), std::sync::atomic::Ordering::SeqCst);
+                w.slot.store(bank.slot(), std::sync::atomic::Ordering::SeqCst);
                 drop(w);
                 self.scheduler.read().unwrap()
             } else {

From e13c590a7ddd40967d1df83c12cd6a455d129c22 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 16 Sep 2022 14:14:04 +0900
Subject: [PATCH 2709/3199] save

---
 runtime/src/bank.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 305a25dcbaea45..f7e001cdf0925a 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -6572,7 +6572,7 @@ impl Bank {
                 drop(w);
                 self.scheduler.read().unwrap()
             } else {
-                assert_eq!(bank.slot(), r.as_ref().unwrap().slot.as_ref().unwrap().load(std::sync::atomic::Ordering::SeqCst));
+                assert_eq!(bank.slot(), r.as_ref().unwrap().slot.load(std::sync::atomic::Ordering::SeqCst));
                 r
             }
         };

From b7dc3506bf54de1a238aed45fd8df208a451f6f9 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 16 Sep 2022 14:14:21 +0900
Subject: [PATCH 2710/3199] save

---
 runtime/src/bank.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index f7e001cdf0925a..2216c7f6973415 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -6566,7 +6566,7 @@ impl Bank {
                 // this relocking (r=>w) is racy here; we want parking_lot's upgrade; but overwriting should be
                 // safe.
                 let ss = self.scheduler.write().unwrap();
-                let mut w = ss.as_ref().unwrap();
+                let w = ss.as_ref().unwrap();
                 *w.bank.write().unwrap() = Some(Arc::downgrade(&bank));
                 w.slot.store(bank.slot(), std::sync::atomic::Ordering::SeqCst);
                 drop(w);

From 35f00c9fcd33a7667f752fd5d2d3e1f1772d353c Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 16 Sep 2022 14:15:28 +0900
Subject: [PATCH 2711/3199] save

---
 scheduler/src/lib.rs | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 85501da88d4e23..1cf2eeb46475ef 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1766,7 +1766,10 @@ pub struct Checkpoint(pub std::sync::Mutex<usize>, pub std::sync::Condvar);
 
 impl Checkpoint {
     pub fn wait_for_restart(&self) {
+        let current_thread_name = std::thread::current().name().unwrap().to_string();
         let mut remaining_threads_guard = self.0.lock().unwrap();
+        info!("Checkpoint::wait_for_restart: {} is entering at {}", current_thread_name, *remaining_threads_guard);
+
 
         *remaining_threads_guard -= 1;
 

From 09f436d38516897c4b09316019a78eb36757615f Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 16 Sep 2022 14:17:10 +0900
Subject: [PATCH 2712/3199] save

---
 scheduler/src/lib.rs | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 1cf2eeb46475ef..ee4dffd6408dfb 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1768,16 +1768,18 @@ impl Checkpoint {
     pub fn wait_for_restart(&self) {
         let current_thread_name = std::thread::current().name().unwrap().to_string();
         let mut remaining_threads_guard = self.0.lock().unwrap();
-        info!("Checkpoint::wait_for_restart: {} is entering at {}", current_thread_name, *remaining_threads_guard);
-
+        info!("Checkpoint::wait_for_restart: {} is entering at {} -> {}", current_thread_name, *remaining_threads_guard, *remaining_threads_guard - 1);
 
         *remaining_threads_guard -= 1;
 
         if *remaining_threads_guard == 0 {
             drop(remaining_threads_guard);
             self.1.notify_all();
+            info!("Checkpoint::wait_for_restart: {} is notified all others...", current_thread_name);
         } else {
+            info!("Checkpoint::wait_for_restart: {} is paused...", current_thread_name);
             let _ = *self.1.wait_while(remaining_threads_guard, |&mut remaining_threads| remaining_threads == 0).unwrap();
+            info!("Checkpoint::wait_for_restart: {} is started...", current_thread_name);
         }
     }
 

From 49d093ba1104fb303040f2af12e36b7ba74a2f0d Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 16 Sep 2022 14:18:48 +0900
Subject: [PATCH 2713/3199] save

---
 runtime/src/bank.rs | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 2216c7f6973415..577087151d89b2 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1438,6 +1438,7 @@ impl Scheduler {
 
                 if let Some(checkpoint) = maybe_checkpoint {
                     checkpoint.wait_for_restart();
+                    continue;
                 } else {
                     break;
                 }
@@ -1482,7 +1483,7 @@ impl Scheduler {
         //let transaction_sender = self.transaction_sender.take().unwrap();
 
         //drop(transaction_sender);
-        let checkpoint = solana_scheduler::Checkpoint::new(3);
+        let checkpoint = solana_scheduler::Checkpoint::new(2);
         self.transaction_sender.as_ref().unwrap().send(solana_scheduler::SchedulablePayload(solana_scheduler::Flushable::Flush(std::sync::Arc::clone(&checkpoint)))).unwrap();
         checkpoint.wait_for_restart();
         {

From 6df885f647ffeb928d74af7cfd18def787e375d2 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 16 Sep 2022 14:23:31 +0900
Subject: [PATCH 2714/3199] save

---
 scheduler/src/lib.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index ee4dffd6408dfb..d5210d0369da8d 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1762,7 +1762,7 @@ pub struct ExecutablePayload(pub Box<ExecutionEnvironment>);
 pub struct UnlockablePayload(pub Box<ExecutionEnvironment>);
 pub struct ExaminablePayload(pub Box<ExecutionEnvironment>);
 
-pub struct Checkpoint(pub std::sync::Mutex<usize>, pub std::sync::Condvar);
+pub struct Checkpoint(std::sync::Mutex<usize>, std::sync::Condvar);
 
 impl Checkpoint {
     pub fn wait_for_restart(&self) {
@@ -1775,7 +1775,7 @@ impl Checkpoint {
         if *remaining_threads_guard == 0 {
             drop(remaining_threads_guard);
             self.1.notify_all();
-            info!("Checkpoint::wait_for_restart: {} is notified all others...", current_thread_name);
+            info!("Checkpoint::wait_for_restart: {} notified all others...", current_thread_name);
         } else {
             info!("Checkpoint::wait_for_restart: {} is paused...", current_thread_name);
             let _ = *self.1.wait_while(remaining_threads_guard, |&mut remaining_threads| remaining_threads == 0).unwrap();

From 4f52e595acab3a7b1874ad2d0793f48dd923e207 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 16 Sep 2022 14:31:35 +0900
Subject: [PATCH 2715/3199] save

---
 scheduler/src/lib.rs | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index d5210d0369da8d..ee5c38f26f48db 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1538,6 +1538,7 @@ impl ScheduleStage {
                                from_disconnected = true;
                                from_prev = never;
                                trace!("flushing2..: {:?} {} {} {} {}", (from_disconnected, from_exec_disconnected), runnable_queue.task_count(), contended_count, executing_queue_count, provisioning_tracker_count);
+                               assert!(maybe_checkpoint.is_none());
                                maybe_checkpoint = Some(checkpoint);
                            },
                            Err(_) => {
@@ -1697,6 +1698,7 @@ impl ScheduleStage {
                                assert!(!from_disconnected);
                                from_disconnected = true;
                                from_prev = never;
+                               assert!(maybe_checkpoint.is_none());
                                maybe_checkpoint = Some(checkpoint);
                             },
                             Flushable::Payload(task) => {

From 526f8344744d1f50076acbf204a7cf60a4873822 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 16 Sep 2022 14:58:45 +0900
Subject: [PATCH 2716/3199] save

---
 runtime/src/bank.rs  | 4 ++--
 scheduler/src/lib.rs | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 577087151d89b2..d14e04a527a1f7 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1213,7 +1213,7 @@ impl SchedulerPool {
 
     fn take_from_pool(&mut self) -> Arc<Scheduler> {
         if let Some(scheduler) = self.schedulers.pop() {
-            info!("SchedulerPoom: {} is taken...", scheduler.random_id);
+            info!("SchedulerPool: id_{:016x} is taken...", scheduler.random_id);
             scheduler
         } else {
             self.create();
@@ -1222,7 +1222,7 @@ impl SchedulerPool {
     }
 
     fn return_to_pool(&mut self, scheduler: Arc<Scheduler>) {
-        info!("SchedulerPool: {} is returned...", scheduler.random_id);
+        info!("SchedulerPool: id_{:016x} is returned...", scheduler.random_id);
         assert_eq!(1, Arc::strong_count(&scheduler));
         assert!(scheduler.collected_errors.lock().unwrap().is_empty());
         self.schedulers.push(scheduler);
diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index ee5c38f26f48db..21ac539cea7bf7 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1780,7 +1780,7 @@ impl Checkpoint {
             info!("Checkpoint::wait_for_restart: {} notified all others...", current_thread_name);
         } else {
             info!("Checkpoint::wait_for_restart: {} is paused...", current_thread_name);
-            let _ = *self.1.wait_while(remaining_threads_guard, |&mut remaining_threads| remaining_threads == 0).unwrap();
+            let _ = *self.1.wait_while(remaining_threads_guard, |&mut remaining_threads| remaining_threads > 0).unwrap();
             info!("Checkpoint::wait_for_restart: {} is started...", current_thread_name);
         }
     }

From 727cbd2d21edfdb01ede41588c3bef1c86eff2c3 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 16 Sep 2022 15:10:16 +0900
Subject: [PATCH 2717/3199] save

---
 runtime/src/bank.rs | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index d14e04a527a1f7..e99dad492d8f3d 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1527,6 +1527,15 @@ impl Drop for Scheduler {
     }
 }
 
+impl Drop for SchedulerPool {
+    fn drop(&mut self) {
+        todo!();
+        //info!("Scheduler::drop(): id_{:016x} begin..", self.random_id);
+        //self.gracefully_stop().unwrap();
+        //info!("Scheduler::drop(): id_{:016x} end...", self.random_id);
+    }
+}
+
 /// Manager for the state of all accounts and programs after processing its entries.
 /// AbiExample is needed even without Serialize/Deserialize; actual (de-)serialization
 /// are implemented elsewhere for versioning

From 18d4501021b1f872e746ec3f3db63bca347b7320 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 16 Sep 2022 15:18:39 +0900
Subject: [PATCH 2718/3199] save

---
 runtime/src/bank.rs | 2 --
 1 file changed, 2 deletions(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index e99dad492d8f3d..94bebb9a51abdd 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1471,13 +1471,11 @@ impl Scheduler {
 
 impl Scheduler {
     fn gracefully_stop(&self) -> Result<()> {
-        /*
         if self.graceful_stop_initiated {
             info!("Scheduler::gracefully_stop(): id_{:016x} (skipped..?)", self.random_id);
             return Ok(());
         }
         self.graceful_stop_initiated = true;
-        */
 
         info!("Scheduler::gracefully_stop(): id_{:016x} waiting..", self.random_id);
         //let transaction_sender = self.transaction_sender.take().unwrap();

From effe8c8aaab11b562ad7eb79cdb90d317b8884ac Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 16 Sep 2022 15:20:13 +0900
Subject: [PATCH 2719/3199] save

---
 runtime/src/bank.rs | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 94bebb9a51abdd..1e774381e96046 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1239,7 +1239,7 @@ struct Scheduler {
     error_collector_thread_handle: Option<std::thread::JoinHandle<Result<(Duration, Duration)>>>,
     transaction_sender: Option<crossbeam_channel::Sender<solana_scheduler::SchedulablePayload>>,
     preloader: Arc<solana_scheduler::Preloader>,
-    graceful_stop_initiated: bool,
+    graceful_stop_initiated: AtomicBool,
     collected_errors: Arc<std::sync::Mutex<Vec<Result<()>>>>,
     bank: std::sync::Arc<std::sync::RwLock<std::option::Option<std::sync::Weak<Bank>>>>,
     slot: AtomicU64,
@@ -1471,11 +1471,11 @@ impl Scheduler {
 
 impl Scheduler {
     fn gracefully_stop(&self) -> Result<()> {
-        if self.graceful_stop_initiated {
+        if self.graceful_stop_initiated.load(std::sync::atomic::Ordering::SeqCst); {
             info!("Scheduler::gracefully_stop(): id_{:016x} (skipped..?)", self.random_id);
             return Ok(());
         }
-        self.graceful_stop_initiated = true;
+        self.graceful_stop_initiated.store(true, std::sync::atomic::Ordering::SeqCst);
 
         info!("Scheduler::gracefully_stop(): id_{:016x} waiting..", self.random_id);
         //let transaction_sender = self.transaction_sender.take().unwrap();
@@ -3766,7 +3766,7 @@ impl Bank {
         // committed before this write lock can be obtained here.
         let mut hash = self.hash.write().unwrap();
         if *hash == Hash::default() {
-            assert!(self.scheduler.read().unwrap().as_ref().unwrap().graceful_stop_initiated);
+            assert!(self.scheduler.read().unwrap().as_ref().unwrap().graceful_stop_initiated.load(std::sync::atomic::Ordering::SeqCst);
             // finish up any deferred changes to account state
             self.collect_rent_eagerly(false);
             self.collect_fees();

From 552dc027892bf7c415bf0919b7a31933e604d8cf Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 16 Sep 2022 15:20:25 +0900
Subject: [PATCH 2720/3199] save

---
 runtime/src/bank.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 1e774381e96046..b275587e1edb3d 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -3766,7 +3766,7 @@ impl Bank {
         // committed before this write lock can be obtained here.
         let mut hash = self.hash.write().unwrap();
         if *hash == Hash::default() {
-            assert!(self.scheduler.read().unwrap().as_ref().unwrap().graceful_stop_initiated.load(std::sync::atomic::Ordering::SeqCst);
+            assert!(self.scheduler.read().unwrap().as_ref().unwrap().graceful_stop_initiated.load(std::sync::atomic::Ordering::SeqCst));
             // finish up any deferred changes to account state
             self.collect_rent_eagerly(false);
             self.collect_fees();

From 433645d9fef4caba2afd68d305dbbcb4de078c88 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 16 Sep 2022 15:20:37 +0900
Subject: [PATCH 2721/3199] save

---
 runtime/src/bank.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index b275587e1edb3d..fe93c4563a762f 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1471,7 +1471,7 @@ impl Scheduler {
 
 impl Scheduler {
     fn gracefully_stop(&self) -> Result<()> {
-        if self.graceful_stop_initiated.load(std::sync::atomic::Ordering::SeqCst); {
+        if self.graceful_stop_initiated.load(std::sync::atomic::Ordering::SeqCst) {
             info!("Scheduler::gracefully_stop(): id_{:016x} (skipped..?)", self.random_id);
             return Ok(());
         }

From 97627264c77bffe3731be5b8075fad6d1c3c9301 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 16 Sep 2022 15:21:19 +0900
Subject: [PATCH 2722/3199] save

---
 runtime/src/bank.rs | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index fe93c4563a762f..4b60adc0d8d18f 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1516,6 +1516,10 @@ impl Scheduler {
     }
 }
 
+impl !Default for Scheduler {
+}
+
+
 impl Drop for Scheduler {
     fn drop(&mut self) {
         todo!();

From d670247efd643c1c824577f18c16fde569ed688d Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 16 Sep 2022 15:21:40 +0900
Subject: [PATCH 2723/3199] save

---
 runtime/src/bank.rs | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 4b60adc0d8d18f..fac29b73f28c52 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1,3 +1,5 @@
+#![feature(negative_impls)]
+
 //! The `bank` module tracks client accounts and the progress of on-chain
 //! programs.
 //!

From 4e60a0fef307aef77c9fb6f0849396a0211c983b Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 16 Sep 2022 15:22:02 +0900
Subject: [PATCH 2724/3199] save

---
 runtime/src/lib.rs | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/runtime/src/lib.rs b/runtime/src/lib.rs
index 247df17fe1d675..18c4a0aabb33c0 100644
--- a/runtime/src/lib.rs
+++ b/runtime/src/lib.rs
@@ -1,3 +1,5 @@
+#![feature(negative_impls)]
+
 #![cfg_attr(RUSTC_WITH_SPECIALIZATION, feature(min_specialization))]
 #![allow(clippy::integer_arithmetic)]
 

From bf65896cf823426486486ae565c82ccd6cbf7f4e Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 16 Sep 2022 15:22:30 +0900
Subject: [PATCH 2725/3199] save

---
 runtime/src/bank.rs | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index fac29b73f28c52..f33c3b5f35f5e1 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1,5 +1,3 @@
-#![feature(negative_impls)]
-
 //! The `bank` module tracks client accounts and the progress of on-chain
 //! programs.
 //!
@@ -1518,7 +1516,7 @@ impl Scheduler {
     }
 }
 
-impl !Default for Scheduler {
+impl !Default for Option<Scheduler> {
 }
 
 

From 5cdec69334c660edfe3b217ccbb2402e284683b9 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 16 Sep 2022 15:23:17 +0900
Subject: [PATCH 2726/3199] save

---
 runtime/src/bank.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index f33c3b5f35f5e1..b01a5bafdee0a1 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1516,7 +1516,7 @@ impl Scheduler {
     }
 }
 
-impl !Default for Option<Scheduler> {
+impl !Default for (Option<Scheduler>) {
 }
 
 

From f6255a77d854dc6ec3ec8c8dd5428f969368eebe Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 16 Sep 2022 15:23:27 +0900
Subject: [PATCH 2727/3199] save

---
 runtime/src/bank.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index b01a5bafdee0a1..a66cd025752be2 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1516,7 +1516,7 @@ impl Scheduler {
     }
 }
 
-impl !Default for (Option<Scheduler>) {
+impl !Default for (Option<Scheduler>, usize) {
 }
 
 

From f12284af0963b36ad2f6146492595530c959a040 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 16 Sep 2022 15:24:09 +0900
Subject: [PATCH 2728/3199] save

---
 runtime/src/bank.rs | 6 +-----
 1 file changed, 1 insertion(+), 5 deletions(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index a66cd025752be2..a107f54d6b8b5c 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1516,10 +1516,6 @@ impl Scheduler {
     }
 }
 
-impl !Default for (Option<Scheduler>, usize) {
-}
-
-
 impl Drop for Scheduler {
     fn drop(&mut self) {
         todo!();
@@ -1713,7 +1709,7 @@ pub struct Bank {
 
     pub incremental_snapshot_persistence: Option<BankIncrementalSnapshotPersistence>,
 
-    scheduler: RwLock<Option<Arc<Scheduler>>>,
+    scheduler2: RwLock<Option<Arc<Scheduler>>>,
 }
 
 struct VoteWithStakeDelegations {

From 3cff004d81ec6229e8e7132276b155101a56069f Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 16 Sep 2022 15:25:59 +0900
Subject: [PATCH 2729/3199] save

---
 runtime/src/bank.rs | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index a107f54d6b8b5c..d6c1b358b19157 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1114,7 +1114,7 @@ impl PartialEq for Bank {
             accounts_data_size_delta_off_chain: _,
             fee_structure: _,
             incremental_snapshot_persistence: _,
-            scheduler: _,
+            scheduler2: _,
             // Ignore new fields explicitly if they do not impact PartialEq.
             // Adding ".." will remove compile-time checks that if a new field
             // is added to the struct, this ParitalEq is accordingly updated.
@@ -1900,7 +1900,7 @@ impl Bank {
             accounts_data_size_delta_on_chain: AtomicI64::new(0),
             accounts_data_size_delta_off_chain: AtomicI64::new(0),
             fee_structure: FeeStructure::default(),
-            scheduler: RwLock::new(Some(SCHEDULER_POOL.lock().unwrap().take_from_pool())),
+            scheduler2: RwLock::new(Some(SCHEDULER_POOL.lock().unwrap().take_from_pool())),
         };
 
         let accounts_data_size_initial = bank.get_total_accounts_stats().unwrap().data_len as u64;
@@ -2140,7 +2140,7 @@ impl Bank {
             measure!(parent.feature_set.clone(), "feature_set_creation");
 
         let accounts_data_size_initial = parent.load_accounts_data_size();
-        let scheduler = Default::default();
+        let scheduler = RwLock::new(Some(SCHEDULER_POOL.lock().unwrap().take_from_pool())); // Default::default();
 
         let mut new = Bank {
             incremental_snapshot_persistence: None,
@@ -2221,7 +2221,7 @@ impl Bank {
             accounts_data_size_delta_on_chain: AtomicI64::new(0),
             accounts_data_size_delta_off_chain: AtomicI64::new(0),
             fee_structure: parent.fee_structure.clone(),
-            scheduler
+            scheduler2: scheduler,
         };
 
         let (_, ancestors_time) = measure!(
@@ -2566,7 +2566,7 @@ impl Bank {
             accounts_data_size_delta_on_chain: AtomicI64::new(0),
             accounts_data_size_delta_off_chain: AtomicI64::new(0),
             fee_structure: FeeStructure::default(),
-            scheduler: Default::default(),
+            scheduler2: RwLock::new(Some(SCHEDULER_POOL.lock().unwrap().take_from_pool())); // Default::default();Default::default(),
         };
         bank.finish_init(
             genesis_config,

From 8c903eff8b0e8cf110239510cf06eaa342cd210b Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 16 Sep 2022 15:26:11 +0900
Subject: [PATCH 2730/3199] save

---
 runtime/src/bank.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index d6c1b358b19157..4fd9547f80d9ba 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -2566,7 +2566,7 @@ impl Bank {
             accounts_data_size_delta_on_chain: AtomicI64::new(0),
             accounts_data_size_delta_off_chain: AtomicI64::new(0),
             fee_structure: FeeStructure::default(),
-            scheduler2: RwLock::new(Some(SCHEDULER_POOL.lock().unwrap().take_from_pool())); // Default::default();Default::default(),
+            scheduler2: RwLock::new(Some(SCHEDULER_POOL.lock().unwrap().take_from_pool())), // Default::default();Default::default(),
         };
         bank.finish_init(
             genesis_config,

From 36f68a984545750090afde59327670cefcc4cd4a Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 16 Sep 2022 15:27:11 +0900
Subject: [PATCH 2731/3199] save

---
 runtime/src/bank.rs | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 4fd9547f80d9ba..f11a2a3ad9a748 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -3766,7 +3766,7 @@ impl Bank {
         // committed before this write lock can be obtained here.
         let mut hash = self.hash.write().unwrap();
         if *hash == Hash::default() {
-            assert!(self.scheduler.read().unwrap().as_ref().unwrap().graceful_stop_initiated.load(std::sync::atomic::Ordering::SeqCst));
+            assert!(self.scheduler2.read().unwrap().as_ref().unwrap().graceful_stop_initiated.load(std::sync::atomic::Ordering::SeqCst));
             // finish up any deferred changes to account state
             self.collect_rent_eagerly(false);
             self.collect_fees();
@@ -4207,7 +4207,7 @@ impl Bank {
         if maybe_last_error.is_err() {
             warn!("register_recent_blockhash: carrying over this error: {:?}", maybe_last_error);
             //new_scheduler.collected_errors.lock().unwrap().push(maybe_last_error);
-            self.scheduler.write().unwrap().as_ref().unwrap().collected_errors.lock().unwrap().push(maybe_last_error);
+            self.scheduler2.write().unwrap().as_ref().unwrap().collected_errors.lock().unwrap().push(maybe_last_error);
         }
         //*self.scheduler.write().unwrap() = new_scheduler;
 
@@ -6567,18 +6567,18 @@ impl Bank {
         assert_eq!(bank.slot(), batch.bank().slot());
 
         let s = {
-            let r = self.scheduler.read().unwrap();
+            let r = self.scheduler2.read().unwrap();
 
             if r.as_ref().unwrap().bank.read().unwrap().is_none() {
                 drop(r);
                 // this relocking (r=>w) is racy here; we want parking_lot's upgrade; but overwriting should be
                 // safe.
-                let ss = self.scheduler.write().unwrap();
+                let ss = self.scheduler2.write().unwrap();
                 let w = ss.as_ref().unwrap();
                 *w.bank.write().unwrap() = Some(Arc::downgrade(&bank));
                 w.slot.store(bank.slot(), std::sync::atomic::Ordering::SeqCst);
                 drop(w);
-                self.scheduler.read().unwrap()
+                self.scheduler2.read().unwrap()
             } else {
                 assert_eq!(bank.slot(), r.as_ref().unwrap().slot.load(std::sync::atomic::Ordering::SeqCst));
                 r
@@ -6594,7 +6594,7 @@ impl Bank {
     pub fn handle_aborted_transactions(
         &self,
     ) -> Vec<Result<()>> {
-        let s = self.scheduler.read().unwrap();
+        let s = self.scheduler2.read().unwrap();
         let scheduler = s.as_ref().unwrap();
         scheduler.handle_aborted_executions()
     }

From af854beadc7bd1c9f9b56368067252844ae0132e Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 16 Sep 2022 15:27:35 +0900
Subject: [PATCH 2732/3199] save

---
 runtime/src/bank.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index f11a2a3ad9a748..53b7d18d28a9b6 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -8248,7 +8248,7 @@ impl Bank {
     }
 
     pub fn wait_for_scheduler(&self) -> Result<()> {
-        let scheduler = self.scheduler.write().unwrap().take().unwrap();
+        let scheduler = self.scheduler2.write().unwrap().take().unwrap();
         scheduler.gracefully_stop().unwrap();
         let e = scheduler.handle_aborted_executions().into_iter().next().unwrap_or(Ok(()));
         SCHEDULER_POOL.lock().unwrap().return_to_pool(scheduler);

From 4d7960a8ea3d5d6560b88655c46a438a86283105 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 16 Sep 2022 15:37:29 +0900
Subject: [PATCH 2733/3199] save

---
 runtime/src/bank.rs | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 53b7d18d28a9b6..dc02502818a11d 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1518,7 +1518,8 @@ impl Scheduler {
 
 impl Drop for Scheduler {
     fn drop(&mut self) {
-        todo!();
+        let current_thread_name = std::thread::current().name().unwrap().to_string();
+        warn!("Scheduler::drop() by {}...", current_thread_name);
         //info!("Scheduler::drop(): id_{:016x} begin..", self.random_id);
         //self.gracefully_stop().unwrap();
         //info!("Scheduler::drop(): id_{:016x} end...", self.random_id);
@@ -1527,7 +1528,8 @@ impl Drop for Scheduler {
 
 impl Drop for SchedulerPool {
     fn drop(&mut self) {
-        todo!();
+        let current_thread_name = std::thread::current().name().unwrap().to_string();
+        warn!("SchedulerPool::drop() by {}...", current_thread_name);
         //info!("Scheduler::drop(): id_{:016x} begin..", self.random_id);
         //self.gracefully_stop().unwrap();
         //info!("Scheduler::drop(): id_{:016x} end...", self.random_id);

From 5fd27c380945a049e7df5390f7e131fdee2a4588 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 16 Sep 2022 15:46:57 +0900
Subject: [PATCH 2734/3199] save

---
 runtime/src/bank.rs | 14 +++++++++-----
 1 file changed, 9 insertions(+), 5 deletions(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index dc02502818a11d..61ee9ddc3a5d8a 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -8250,11 +8250,15 @@ impl Bank {
     }
 
     pub fn wait_for_scheduler(&self) -> Result<()> {
-        let scheduler = self.scheduler2.write().unwrap().take().unwrap();
-        scheduler.gracefully_stop().unwrap();
-        let e = scheduler.handle_aborted_executions().into_iter().next().unwrap_or(Ok(()));
-        SCHEDULER_POOL.lock().unwrap().return_to_pool(scheduler);
-        e
+        let s = self.scheduler2.write().unwrap().take();
+        if let Some(scheduler) = s {
+            scheduler.gracefully_stop().unwrap();
+            let e = scheduler.handle_aborted_executions().into_iter().next().unwrap_or(Ok(()));
+            SCHEDULER_POOL.lock().unwrap().return_to_pool(scheduler);
+            e
+        } else {
+            Ok(())
+        }
     }
 }
 

From df23f2299637598334a5e4e6c3dfd4ab02df4d92 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 16 Sep 2022 15:48:55 +0900
Subject: [PATCH 2735/3199] save

---
 runtime/src/bank.rs | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 61ee9ddc3a5d8a..9d5a8f00474c7d 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -8257,6 +8257,9 @@ impl Bank {
             SCHEDULER_POOL.lock().unwrap().return_to_pool(scheduler);
             e
         } else {
+            let current_thread_name = std::thread::current().name().unwrap().to_string();
+            warn!("Bank::wait_for_scheduler() skipped by {} ...", current_thread_name);
+
             Ok(())
         }
     }

From a7028b2336e080984edfb114207fb3dd34101194 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 16 Sep 2022 15:57:13 +0900
Subject: [PATCH 2736/3199] save

---
 runtime/src/bank.rs | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 9d5a8f00474c7d..ee01f02a9dd1a9 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -4200,6 +4200,9 @@ impl Bank {
         info!("register_recent_blockhash: slot: {} reinitializing the scheduler: start", self.slot());
 
         let maybe_last_error = self.wait_for_scheduler();
+        let scheduler = SCHEDULER_POOL.lock().unwrap().take_from_pool();
+        let s2 = self.scheduler2.write().unwrap();
+        *s2 = Some(scheduler);
 
         // Only acquire the write lock for the blockhash queue on block boundaries because
         // readers can starve this write lock acquisition and ticks would be slowed down too
@@ -4209,7 +4212,7 @@ impl Bank {
         if maybe_last_error.is_err() {
             warn!("register_recent_blockhash: carrying over this error: {:?}", maybe_last_error);
             //new_scheduler.collected_errors.lock().unwrap().push(maybe_last_error);
-            self.scheduler2.write().unwrap().as_ref().unwrap().collected_errors.lock().unwrap().push(maybe_last_error);
+            s2.as_ref().unwrap().collected_errors.lock().unwrap().push(maybe_last_error);
         }
         //*self.scheduler.write().unwrap() = new_scheduler;
 

From ee16b7f7bdd2e5656eba66f6c2524701a3c766b6 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 16 Sep 2022 15:57:34 +0900
Subject: [PATCH 2737/3199] save

---
 runtime/src/bank.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index ee01f02a9dd1a9..48bc1edc037d03 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -4201,7 +4201,7 @@ impl Bank {
 
         let maybe_last_error = self.wait_for_scheduler();
         let scheduler = SCHEDULER_POOL.lock().unwrap().take_from_pool();
-        let s2 = self.scheduler2.write().unwrap();
+        let mut s2 = self.scheduler2.write().unwrap();
         *s2 = Some(scheduler);
 
         // Only acquire the write lock for the blockhash queue on block boundaries because

From 6ec2da4b1cd534f9e77e66740e4329f941e9201f Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 16 Sep 2022 15:59:27 +0900
Subject: [PATCH 2738/3199] save

---
 runtime/src/bank.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 48bc1edc037d03..8bb795451f1cf1 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -3768,7 +3768,7 @@ impl Bank {
         // committed before this write lock can be obtained here.
         let mut hash = self.hash.write().unwrap();
         if *hash == Hash::default() {
-            assert!(self.scheduler2.read().unwrap().as_ref().unwrap().graceful_stop_initiated.load(std::sync::atomic::Ordering::SeqCst));
+            assert!(self.scheduler2.read().unwrap().as_ref().unwrap().is_none());
             // finish up any deferred changes to account state
             self.collect_rent_eagerly(false);
             self.collect_fees();

From 93836e3b830d2b722407e6c08fdf3762c2d3e5ea Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 16 Sep 2022 15:59:57 +0900
Subject: [PATCH 2739/3199] save

---
 runtime/src/bank.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 8bb795451f1cf1..1becc336268f3a 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -3768,7 +3768,7 @@ impl Bank {
         // committed before this write lock can be obtained here.
         let mut hash = self.hash.write().unwrap();
         if *hash == Hash::default() {
-            assert!(self.scheduler2.read().unwrap().as_ref().unwrap().is_none());
+            assert!(self.scheduler2.read().unwrap().is_none());
             // finish up any deferred changes to account state
             self.collect_rent_eagerly(false);
             self.collect_fees();

From 47cad26bdfcbc319721369825cf8d138651c4ba8 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 16 Sep 2022 16:32:44 +0900
Subject: [PATCH 2740/3199] save

---
 runtime/src/bank.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 1becc336268f3a..ffdd9b819db21e 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -8257,7 +8257,7 @@ impl Bank {
         if let Some(scheduler) = s {
             scheduler.gracefully_stop().unwrap();
             let e = scheduler.handle_aborted_executions().into_iter().next().unwrap_or(Ok(()));
-            SCHEDULER_POOL.lock().unwrap().return_to_pool(scheduler);
+            SCHEDULER_POOL.lock().unwrap().as_ref().return_to_pool(scheduler);
             e
         } else {
             let current_thread_name = std::thread::current().name().unwrap().to_string();

From 9131051514a6a1e326ad7e67139ebee1e59574ba Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 16 Sep 2022 16:36:34 +0900
Subject: [PATCH 2741/3199] save

---
 runtime/src/bank.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index ffdd9b819db21e..bd35651ca5c195 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1229,7 +1229,7 @@ impl SchedulerPool {
     }
 }
 
-const SCHEDULER_POOL: std::sync::Mutex<SchedulerPool> = std::sync::Mutex::new(SchedulerPool::new());
+const SCHEDULER_POOL: std::sync::Arc<std::sync::Mutex<SchedulerPool>> = std::sync::Arc::new(std::sync::Mutex::new(SchedulerPool::new()));
 
 #[derive(Debug)]
 struct Scheduler {
@@ -8257,7 +8257,7 @@ impl Bank {
         if let Some(scheduler) = s {
             scheduler.gracefully_stop().unwrap();
             let e = scheduler.handle_aborted_executions().into_iter().next().unwrap_or(Ok(()));
-            SCHEDULER_POOL.lock().unwrap().as_ref().return_to_pool(scheduler);
+            SCHEDULER_POOL.lock().unwrap().return_to_pool(scheduler);
             e
         } else {
             let current_thread_name = std::thread::current().name().unwrap().to_string();

From 4298bf6cacc272fefc68804a04e98da8c2e69950 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 16 Sep 2022 16:38:38 +0900
Subject: [PATCH 2742/3199] save

---
 runtime/src/bank.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index bd35651ca5c195..b2fd166993a197 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1229,7 +1229,7 @@ impl SchedulerPool {
     }
 }
 
-const SCHEDULER_POOL: std::sync::Arc<std::sync::Mutex<SchedulerPool>> = std::sync::Arc::new(std::sync::Mutex::new(SchedulerPool::new()));
+static SCHEDULER_POOL: std::sync::Arc<std::sync::Mutex<SchedulerPool> = std::sync::Mutex::new(SchedulerPool::new());
 
 #[derive(Debug)]
 struct Scheduler {

From 4ec1cd38706e33feb30e2881d760b5d865367a48 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 16 Sep 2022 16:38:51 +0900
Subject: [PATCH 2743/3199] save

---
 runtime/src/bank.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index b2fd166993a197..1f3fe7b6df3605 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1229,7 +1229,7 @@ impl SchedulerPool {
     }
 }
 
-static SCHEDULER_POOL: std::sync::Arc<std::sync::Mutex<SchedulerPool> = std::sync::Mutex::new(SchedulerPool::new());
+static SCHEDULER_POOL: std::sync::Mutex<SchedulerPool> = std::sync::Mutex::new(SchedulerPool::new());
 
 #[derive(Debug)]
 struct Scheduler {

From df64ef1b8f2361fbc9bf8d2995fb42dd8a19e4dd Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 16 Sep 2022 16:52:45 +0900
Subject: [PATCH 2744/3199] save

---
 runtime/src/bank.rs | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 1f3fe7b6df3605..7cf6fd1f22c8ff 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1213,7 +1213,7 @@ impl SchedulerPool {
 
     fn take_from_pool(&mut self) -> Arc<Scheduler> {
         if let Some(scheduler) = self.schedulers.pop() {
-            info!("SchedulerPool: id_{:016x} is taken...", scheduler.random_id);
+            info!("SchedulerPool: id_{:016x} is taken... len: {} => {}", scheduler.random_id, self.schedulers.len() + 1, self.schedulers.len());
             scheduler
         } else {
             self.create();
@@ -1222,9 +1222,13 @@ impl SchedulerPool {
     }
 
     fn return_to_pool(&mut self, scheduler: Arc<Scheduler>) {
-        info!("SchedulerPool: id_{:016x} is returned...", scheduler.random_id);
+        info!("SchedulerPool: id_{:016x} is returned... len: {} => {}", scheduler.random_id, self.schedulers.len(), self.schedulers.len() + 1);
         assert_eq!(1, Arc::strong_count(&scheduler));
         assert!(scheduler.collected_errors.lock().unwrap().is_empty());
+        assert!(scheduler.graceful_stop_initiated.load(std::sync::atomic::Ordering::SeqCst);
+
+        scheduler.graceful_stop_initiated.store(false, std::sync::atomic::Ordering::SeqCst);
+
         self.schedulers.push(scheduler);
     }
 }

From d1b877a1950d691fbcb2fe6f1fb2145e3c2c752a Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 16 Sep 2022 16:52:57 +0900
Subject: [PATCH 2745/3199] save

---
 runtime/src/bank.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 7cf6fd1f22c8ff..a9b82f7017885f 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1225,7 +1225,7 @@ impl SchedulerPool {
         info!("SchedulerPool: id_{:016x} is returned... len: {} => {}", scheduler.random_id, self.schedulers.len(), self.schedulers.len() + 1);
         assert_eq!(1, Arc::strong_count(&scheduler));
         assert!(scheduler.collected_errors.lock().unwrap().is_empty());
-        assert!(scheduler.graceful_stop_initiated.load(std::sync::atomic::Ordering::SeqCst);
+        assert!(scheduler.graceful_stop_initiated.load(std::sync::atomic::Ordering::SeqCst));
 
         scheduler.graceful_stop_initiated.store(false, std::sync::atomic::Ordering::SeqCst);
 

From b1e156b1d06496e75f63fd87ae16b32f80cb692a Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 16 Sep 2022 17:08:09 +0900
Subject: [PATCH 2746/3199] save

---
 runtime/src/bank.rs | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index a9b82f7017885f..9d599c1b04f8d6 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1251,6 +1251,7 @@ struct Scheduler {
 
 impl Scheduler {
     fn schedule(&self, sanitized_tx: &SanitizedTransaction, index: usize) {
+        info!("Scheduler::schedule()");
         #[derive(Clone, Copy, Debug)]
         struct NotAtTopOfScheduleThread;
         unsafe impl solana_scheduler::NotAtScheduleThread for NotAtTopOfScheduleThread {}
@@ -1524,6 +1525,7 @@ impl Drop for Scheduler {
     fn drop(&mut self) {
         let current_thread_name = std::thread::current().name().unwrap().to_string();
         warn!("Scheduler::drop() by {}...", current_thread_name);
+        todo!();
         //info!("Scheduler::drop(): id_{:016x} begin..", self.random_id);
         //self.gracefully_stop().unwrap();
         //info!("Scheduler::drop(): id_{:016x} end...", self.random_id);
@@ -1534,6 +1536,7 @@ impl Drop for SchedulerPool {
     fn drop(&mut self) {
         let current_thread_name = std::thread::current().name().unwrap().to_string();
         warn!("SchedulerPool::drop() by {}...", current_thread_name);
+        todo!();
         //info!("Scheduler::drop(): id_{:016x} begin..", self.random_id);
         //self.gracefully_stop().unwrap();
         //info!("Scheduler::drop(): id_{:016x} end...", self.random_id);
@@ -6574,6 +6577,7 @@ impl Bank {
         transaction_indexes: &Vec<usize>,
     ) {
         assert_eq!(bank.slot(), batch.bank().slot());
+        info!("schedule_and_commit_transactions()");
 
         let s = {
             let r = self.scheduler2.read().unwrap();

From e95eb2aaec29933014f89ac6fdbde4a6a8d8aceb Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 16 Sep 2022 17:09:21 +0900
Subject: [PATCH 2747/3199] save

---
 runtime/src/bank.rs | 1 +
 1 file changed, 1 insertion(+)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 9d599c1b04f8d6..6e5aa5dd4c41ae 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -6584,6 +6584,7 @@ impl Bank {
 
             if r.as_ref().unwrap().bank.read().unwrap().is_none() {
                 drop(r);
+                info!("reconfiguring scheduler with new bank...");
                 // this relocking (r=>w) is racy here; we want parking_lot's upgrade; but overwriting should be
                 // safe.
                 let ss = self.scheduler2.write().unwrap();

From 86a3c892f68dcfa1bd384020dd3cef66c6752a43 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 16 Sep 2022 17:19:27 +0900
Subject: [PATCH 2748/3199] save

---
 runtime/src/bank.rs | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 6e5aa5dd4c41ae..7ca38bc2d91302 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1389,6 +1389,8 @@ impl Scheduler {
                 //ee.reindex_with_address_book();
                 processed_ee_sender.send(solana_scheduler::UnlockablePayload(ee)).unwrap();
             }
+            todo!();
+
             Ok((started.0.elapsed(), started.1.elapsed()))
         }).unwrap()}).collect();
 
@@ -1411,6 +1413,7 @@ impl Scheduler {
                         }
                         drop(ee);
             }
+            todo!();
 
             Ok((started.0.elapsed(), started.1.elapsed()))
         }).unwrap();
@@ -1453,6 +1456,7 @@ impl Scheduler {
             drop(scheduled_ee_sender);
             drop(processed_ee_receiver);
 
+            todo!();
             Ok((started.0.elapsed(), started.1.elapsed()))
         }).unwrap();
 
@@ -6592,6 +6596,7 @@ impl Bank {
                 *w.bank.write().unwrap() = Some(Arc::downgrade(&bank));
                 w.slot.store(bank.slot(), std::sync::atomic::Ordering::SeqCst);
                 drop(w);
+                info!("reconfiguring is done");
                 self.scheduler2.read().unwrap()
             } else {
                 assert_eq!(bank.slot(), r.as_ref().unwrap().slot.load(std::sync::atomic::Ordering::SeqCst));

From 4a3bc3343e686b4fe7992debe5538b810079fefd Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 16 Sep 2022 17:20:57 +0900
Subject: [PATCH 2749/3199] save

---
 runtime/src/bank.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 7ca38bc2d91302..f302915b40cf19 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -8267,7 +8267,7 @@ impl Bank {
     }
 
     pub fn wait_for_scheduler(&self) -> Result<()> {
-        let s = self.scheduler2.write().unwrap().take();
+        let s: usize = self.scheduler2.write().unwrap().take();
         if let Some(scheduler) = s {
             scheduler.gracefully_stop().unwrap();
             let e = scheduler.handle_aborted_executions().into_iter().next().unwrap_or(Ok(()));

From dc8658bdc66f1d307a8dd736564679a968f24869 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 16 Sep 2022 17:21:26 +0900
Subject: [PATCH 2750/3199] save

---
 runtime/src/bank.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index f302915b40cf19..63647bb481da7d 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -8267,7 +8267,7 @@ impl Bank {
     }
 
     pub fn wait_for_scheduler(&self) -> Result<()> {
-        let s: usize = self.scheduler2.write().unwrap().take();
+        let s: Option<usize> = self.scheduler2.write().unwrap().take();
         if let Some(scheduler) = s {
             scheduler.gracefully_stop().unwrap();
             let e = scheduler.handle_aborted_executions().into_iter().next().unwrap_or(Ok(()));

From bb7f92080d2d1a74eef982214215f883714db1e3 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 16 Sep 2022 17:21:45 +0900
Subject: [PATCH 2751/3199] save

---
 runtime/src/bank.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 63647bb481da7d..78182337ee5a82 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -8267,7 +8267,7 @@ impl Bank {
     }
 
     pub fn wait_for_scheduler(&self) -> Result<()> {
-        let s: Option<usize> = self.scheduler2.write().unwrap().take();
+        let s: Option<Arc<Scheduler>> = self.scheduler2.write().unwrap().take();
         if let Some(scheduler) = s {
             scheduler.gracefully_stop().unwrap();
             let e = scheduler.handle_aborted_executions().into_iter().next().unwrap_or(Ok(()));

From 2bfe7a54b7d3ad0ed76618ed19319f8f64c3369f Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 16 Sep 2022 17:21:49 +0900
Subject: [PATCH 2752/3199] save

---
 runtime/src/bank.rs | 1 +
 1 file changed, 1 insertion(+)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 78182337ee5a82..a9ce48b63e01a2 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -8268,6 +8268,7 @@ impl Bank {
 
     pub fn wait_for_scheduler(&self) -> Result<()> {
         let s: Option<Arc<Scheduler>> = self.scheduler2.write().unwrap().take();
+
         if let Some(scheduler) = s {
             scheduler.gracefully_stop().unwrap();
             let e = scheduler.handle_aborted_executions().into_iter().next().unwrap_or(Ok(()));

From 57a710036140191183ed7535fd07d6d28ef7f5a9 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 16 Sep 2022 17:29:21 +0900
Subject: [PATCH 2753/3199] save

---
 runtime/src/bank.rs | 1 +
 1 file changed, 1 insertion(+)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index a9ce48b63e01a2..684ae6e1dbb552 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -6596,6 +6596,7 @@ impl Bank {
                 *w.bank.write().unwrap() = Some(Arc::downgrade(&bank));
                 w.slot.store(bank.slot(), std::sync::atomic::Ordering::SeqCst);
                 drop(w);
+                drop(ss);
                 info!("reconfiguring is done");
                 self.scheduler2.read().unwrap()
             } else {

From c7844bb2711c8ccec698b60031bcd305b5d3f3cd Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 16 Sep 2022 21:43:52 +0900
Subject: [PATCH 2754/3199] save

---
 runtime/src/bank.rs | 13 ++++++++++---
 1 file changed, 10 insertions(+), 3 deletions(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 684ae6e1dbb552..c0a595c9330b87 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1525,6 +1525,7 @@ impl Scheduler {
     }
 }
 
+/*
 impl Drop for Scheduler {
     fn drop(&mut self) {
         let current_thread_name = std::thread::current().name().unwrap().to_string();
@@ -1535,6 +1536,7 @@ impl Drop for Scheduler {
         //info!("Scheduler::drop(): id_{:016x} end...", self.random_id);
     }
 }
+*/
 
 impl Drop for SchedulerPool {
     fn drop(&mut self) {
@@ -4210,7 +4212,7 @@ impl Bank {
     pub fn register_recent_blockhash(&self, blockhash: &Hash) {
         info!("register_recent_blockhash: slot: {} reinitializing the scheduler: start", self.slot());
 
-        let maybe_last_error = self.wait_for_scheduler();
+        let maybe_last_error = self.wait_for_scheduler(false);
         let scheduler = SCHEDULER_POOL.lock().unwrap().take_from_pool();
         let mut s2 = self.scheduler2.write().unwrap();
         *s2 = Some(scheduler);
@@ -8267,7 +8269,7 @@ impl Bank {
         total_accounts_stats
     }
 
-    pub fn wait_for_scheduler(&self) -> Result<()> {
+    pub fn wait_for_scheduler(&self, via_drop: bool) -> Result<()> {
         let s: Option<Arc<Scheduler>> = self.scheduler2.write().unwrap().take();
 
         if let Some(scheduler) = s {
@@ -8277,7 +8279,7 @@ impl Bank {
             e
         } else {
             let current_thread_name = std::thread::current().name().unwrap().to_string();
-            warn!("Bank::wait_for_scheduler() skipped by {} ...", current_thread_name);
+            warn!("Bank::wait_for_scheduler(via_drop: {}) skipped by {} ...", via_drop, current_thread_name);
 
             Ok(())
         }
@@ -8408,6 +8410,11 @@ pub struct TotalAccountsStats {
 
 impl Drop for Bank {
     fn drop(&mut self) {
+        let r = self.wait_for_scheduler(true).unwrap();
+        if let Err(err) = r {
+            warn!("Bank::drop(): discarding error from scheduler: {:?}", err);
+        }
+
         if let Some(drop_callback) = self.drop_callback.read().unwrap().0.as_ref() {
             drop_callback.callback(self);
         } else {

From 2f393b85472e4748d1d3dc6802916dced26e2e06 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 16 Sep 2022 21:44:19 +0900
Subject: [PATCH 2755/3199] save

---
 runtime/src/bank.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index c0a595c9330b87..7a23c9998a5d63 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -8410,7 +8410,7 @@ pub struct TotalAccountsStats {
 
 impl Drop for Bank {
     fn drop(&mut self) {
-        let r = self.wait_for_scheduler(true).unwrap();
+        let r = self.wait_for_scheduler(true);
         if let Err(err) = r {
             warn!("Bank::drop(): discarding error from scheduler: {:?}", err);
         }

From 73633b3ad8961fee0e7659b62ce7da70d51a6b27 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 16 Sep 2022 21:45:23 +0900
Subject: [PATCH 2756/3199] save

---
 runtime/src/bank.rs | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 7a23c9998a5d63..2b78b81cc4f42e 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -8412,7 +8412,9 @@ impl Drop for Bank {
     fn drop(&mut self) {
         let r = self.wait_for_scheduler(true);
         if let Err(err) = r {
-            warn!("Bank::drop(): discarding error from scheduler: {:?}", err);
+            warn!("Bank::drop(): slot: {} discarding error from scheduler: {:?}", self.slot(), err);
+        } else {
+            trace!("Bank::drop(): slot: {} scheduler is returned to the pool", self.slot());
         }
 
         if let Some(drop_callback) = self.drop_callback.read().unwrap().0.as_ref() {

From b539a8cfaf442e38cc72e95518f81516d01fcb77 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 16 Sep 2022 21:50:41 +0900
Subject: [PATCH 2757/3199] save

---
 runtime/src/bank.rs | 12 +++++++-----
 1 file changed, 7 insertions(+), 5 deletions(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 2b78b81cc4f42e..6741838ed992be 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1251,7 +1251,7 @@ struct Scheduler {
 
 impl Scheduler {
     fn schedule(&self, sanitized_tx: &SanitizedTransaction, index: usize) {
-        info!("Scheduler::schedule()");
+        trace!("Scheduler::schedule()");
         #[derive(Clone, Copy, Debug)]
         struct NotAtTopOfScheduleThread;
         unsafe impl solana_scheduler::NotAtScheduleThread for NotAtTopOfScheduleThread {}
@@ -6583,14 +6583,14 @@ impl Bank {
         transaction_indexes: &Vec<usize>,
     ) {
         assert_eq!(bank.slot(), batch.bank().slot());
-        info!("schedule_and_commit_transactions()");
+        trace!("schedule_and_commit_transactions(): [{:?}; {}]", transaction_indexes.iter().next(), transaction_indexes.len());
 
         let s = {
             let r = self.scheduler2.read().unwrap();
 
             if r.as_ref().unwrap().bank.read().unwrap().is_none() {
                 drop(r);
-                info!("reconfiguring scheduler with new bank...");
+                //info!("reconfiguring scheduler with new bank...");
                 // this relocking (r=>w) is racy here; we want parking_lot's upgrade; but overwriting should be
                 // safe.
                 let ss = self.scheduler2.write().unwrap();
@@ -6599,8 +6599,10 @@ impl Bank {
                 w.slot.store(bank.slot(), std::sync::atomic::Ordering::SeqCst);
                 drop(w);
                 drop(ss);
-                info!("reconfiguring is done");
-                self.scheduler2.read().unwrap()
+
+                let s = self.scheduler2.read().unwrap();
+                info!("reconfigured scheduler to the bank slot: {}", self.slot());
+                s
             } else {
                 assert_eq!(bank.slot(), r.as_ref().unwrap().slot.load(std::sync::atomic::Ordering::SeqCst));
                 r

From e02bba367a1d466c5846e52273735fe3f40e0a09 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 16 Sep 2022 21:57:25 +0900
Subject: [PATCH 2758/3199] save

---
 ledger/src/blockstore_processor.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/ledger/src/blockstore_processor.rs b/ledger/src/blockstore_processor.rs
index 9690aae9792ccf..f181d89a0c9f08 100644
--- a/ledger/src/blockstore_processor.rs
+++ b/ledger/src/blockstore_processor.rs
@@ -1333,7 +1333,7 @@ fn process_bank_0(
         &mut ExecuteTimings::default(),
     )
     .expect("Failed to process bank 0 from ledger. Did you forget to provide a snapshot?");
-    bank0.wait_for_scheduler().unwrap();
+    bank0.wait_for_scheduler(false).unwrap();
     bank0.freeze();
     if blockstore.is_primary_access() {
         blockstore.insert_bank_hash(bank0.slot(), bank0.hash(), false);
@@ -1682,7 +1682,7 @@ fn process_single_slot(
         err
     })?;
 
-    bank.wait_for_scheduler()?;
+    bank.wait_for_scheduler(false)?;
     bank.freeze(); // all banks handled by this routine are created from complete slots
     if blockstore.is_primary_access() {
         blockstore.insert_bank_hash(bank.slot(), bank.hash(), false);

From b36b8fb647752c745d3e61905d5af816df7c254c Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 16 Sep 2022 22:00:58 +0900
Subject: [PATCH 2759/3199] save

---
 core/src/replay_stage.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/core/src/replay_stage.rs b/core/src/replay_stage.rs
index 09d309de6cdafc..ff31f69fb1f392 100644
--- a/core/src/replay_stage.rs
+++ b/core/src/replay_stage.rs
@@ -2472,7 +2472,7 @@ impl ReplayStage {
 
             assert_eq!(bank_slot, bank.slot());
             if bank.is_complete() {
-                let r = bank.wait_for_scheduler();
+                let r = bank.wait_for_scheduler(false);
                 if let Err(err) = r {
                     // Error means the slot needs to be marked as dead
                     Self::mark_dead_slot(

From 19d8b9c63977fa4e8e8edba32a390b0bba8584ee Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 17 Sep 2022 11:53:00 +0900
Subject: [PATCH 2760/3199] save

---
 core/src/replay_stage.rs           |  2 +-
 ledger/src/blockstore_processor.rs |  1 +
 runtime/src/bank.rs                | 12 +++++++-----
 3 files changed, 9 insertions(+), 6 deletions(-)

diff --git a/core/src/replay_stage.rs b/core/src/replay_stage.rs
index ff31f69fb1f392..993e517c2854db 100644
--- a/core/src/replay_stage.rs
+++ b/core/src/replay_stage.rs
@@ -440,7 +440,7 @@ impl ReplayStage {
 
         #[allow(clippy::cognitive_complexity)]
         let t_replay = Builder::new()
-            .name("solReplayStage".to_string())
+            .name("solScReplayStg".to_string())
             .spawn(move || {
                 let verify_recyclers = VerifyRecyclers::default();
                 let _exit = Finalizer::new(exit.clone());
diff --git a/ledger/src/blockstore_processor.rs b/ledger/src/blockstore_processor.rs
index f181d89a0c9f08..de602aeb48dcb6 100644
--- a/ledger/src/blockstore_processor.rs
+++ b/ledger/src/blockstore_processor.rs
@@ -388,6 +388,7 @@ fn rebatch_transactions<'a>(
     }
 }
 
+#[inline(never)]
 fn execute_batches(
     bank: &Arc<Bank>,
     batches: &[TransactionBatchWithIndexes],
diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 6741838ed992be..d766d415865836 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -8412,11 +8412,13 @@ pub struct TotalAccountsStats {
 
 impl Drop for Bank {
     fn drop(&mut self) {
-        let r = self.wait_for_scheduler(true);
-        if let Err(err) = r {
-            warn!("Bank::drop(): slot: {} discarding error from scheduler: {:?}", self.slot(), err);
-        } else {
-            trace!("Bank::drop(): slot: {} scheduler is returned to the pool", self.slot());
+        if self.scheduler2.read().unwrap().is_some() {
+            let r = self.wait_for_scheduler(true);
+            if let Err(err) = r {
+                warn!("Bank::drop(): slot: {} discarding error from scheduler: {:?}", self.slot(), err);
+            } else {
+                trace!("Bank::drop(): slot: {} scheduler is returned to the pool", self.slot());
+            }
         }
 
         if let Some(drop_callback) = self.drop_callback.read().unwrap().0.as_ref() {

From 0adbad5ae2243e22f0b487f5897bc9bd9baf3629 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 17 Sep 2022 11:58:53 +0900
Subject: [PATCH 2761/3199] save

---
 ledger/src/blockstore_processor.rs | 1 +
 1 file changed, 1 insertion(+)

diff --git a/ledger/src/blockstore_processor.rs b/ledger/src/blockstore_processor.rs
index de602aeb48dcb6..68a6d76c96c23b 100644
--- a/ledger/src/blockstore_processor.rs
+++ b/ledger/src/blockstore_processor.rs
@@ -368,6 +368,7 @@ fn execute_batches_internal(
     })
 }
 
+#[inline(never)]
 fn rebatch_transactions<'a>(
     lock_results: &'a [Result<()>],
     bank: &'a Arc<Bank>,

From e0c5efe7cd00e20ad3c8ba281c7327bbb357dc57 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 17 Sep 2022 13:18:24 +0900
Subject: [PATCH 2762/3199] save

---
 core/src/replay_stage.rs | 2 +-
 runtime/src/bank.rs      | 2 --
 2 files changed, 1 insertion(+), 3 deletions(-)

diff --git a/core/src/replay_stage.rs b/core/src/replay_stage.rs
index 993e517c2854db..b181287a512237 100644
--- a/core/src/replay_stage.rs
+++ b/core/src/replay_stage.rs
@@ -912,7 +912,7 @@ impl ReplayStage {
                     if !did_complete_bank {
                         // only wait for the signal if we did not just process a bank; maybe there are more slots available
 
-                        let timer = Duration::from_millis(100);
+                        let timer = Duration::from_millis(10);
                         let result = ledger_signal_receiver.recv_timeout(timer);
                         match result {
                             Err(RecvTimeoutError::Timeout) => (),
diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index d766d415865836..54ea4409be34e1 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1525,7 +1525,6 @@ impl Scheduler {
     }
 }
 
-/*
 impl Drop for Scheduler {
     fn drop(&mut self) {
         let current_thread_name = std::thread::current().name().unwrap().to_string();
@@ -1536,7 +1535,6 @@ impl Drop for Scheduler {
         //info!("Scheduler::drop(): id_{:016x} end...", self.random_id);
     }
 }
-*/
 
 impl Drop for SchedulerPool {
     fn drop(&mut self) {

From 6825298235142a817736e9445c2f5114312602ae Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 17 Sep 2022 14:03:51 +0900
Subject: [PATCH 2763/3199] save

---
 runtime/src/bank.rs  |  2 +-
 scheduler/src/lib.rs | 20 ++++++++++----------
 2 files changed, 11 insertions(+), 11 deletions(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 54ea4409be34e1..4c0daac69dab8d 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1386,7 +1386,7 @@ impl Scheduler {
 
                 ee.execution_result = Some(tx_result);
 
-                //ee.reindex_with_address_book();
+                ee.reindex_with_address_book();
                 processed_ee_sender.send(solana_scheduler::UnlockablePayload(ee)).unwrap();
             }
             todo!();
diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 21ac539cea7bf7..d68e8f0100c8f7 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -19,7 +19,7 @@ type PageRcInner<T> = std::rc::Rc<T>;
 unsafe impl Send for PageRc {}
 */
 
-type PageRcInner = triomphe::Arc<(std::cell::RefCell<Page>, TaskIds)>;
+type PageRcInner = triomphe::Arc<(std::cell::RefCell<Page>, TaskIds, AtomicUsize)>;
 
 #[derive(Debug, Clone)]
 pub struct PageRc(PageRcInner);
@@ -52,7 +52,7 @@ impl ExecutionEnvironment {
     //}
     //
     #[inline(never)]
-    fn reindex_with_address_book<AST: AtScheduleThread>(&mut self, ast: AST) {
+    fn reindex_with_address_book(&mut self) {
         assert!(!self.is_reindexed());
         self.is_reindexed = true;
 
@@ -104,8 +104,7 @@ impl ExecutionEnvironment {
                 });
 
             if should_remove && lock_attempt.requested_usage == RequestedUsage::Writable {
-                let mut page = lock_attempt.target.page_mut(ast);
-                page.contended_write_task_count = page.contended_write_task_count.checked_sub(1).unwrap();
+                lock_attempt.target_contended_write_task_count().fetch_sub(1, std::sync::atomic::Ordering::SeqCst););
             }
         }
     }
@@ -173,6 +172,10 @@ impl LockAttempt {
     pub fn target_contended_unique_weights(&self) -> &TaskIds {
         &self.target.0.1
     }
+
+    pub fn target_contended_write_task_count(&self) -> &AtomicUsize {
+        &self.target.0.2
+    }
 }
 
 type UsageCount = usize;
@@ -243,7 +246,6 @@ pub struct Page {
     next_usage: Usage,
     provisional_task_ids: Vec<triomphe::Arc<ProvisioningTracker>>,
     cu: CU,
-    contended_write_task_count: usize,
     //loaded account from Accounts db
     //comulative_cu for qos; i.e. track serialized cumulative keyed by addresses and bail out block
     //producing as soon as any one of cu from the executing thread reaches to the limit
@@ -257,7 +259,6 @@ impl Page {
             next_usage: Usage::Unused,
             provisional_task_ids: Default::default(),
             cu: Default::default(),
-            contended_write_task_count: Default::default(),
         }
     }
 
@@ -331,7 +332,7 @@ impl AddressBook {
             true
         } else if attempt.target_contended_unique_weights().task_ids.back().unwrap().key() == unique_weight {
             true
-        } else if attempt.requested_usage == RequestedUsage::Readonly && attempt.target.page_mut(ast).contended_write_task_count == 0 {
+        } else if attempt.requested_usage == RequestedUsage::Readonly && attempt.target_contended_write_task_count().load(std::sync::atomic::Ordering::SeqCst) == 0 {
             true
         } else {
             false
@@ -746,8 +747,7 @@ impl Task {
             lock_attempt.target_contended_unique_weights().insert_task(this.unique_weight, Task::clone_in_queue(this));
 
             if lock_attempt.requested_usage == RequestedUsage::Writable {
-                let mut page = lock_attempt.target.page_mut(ast);
-                page.contended_write_task_count = page.contended_write_task_count.checked_add(1).unwrap();
+                lock_attempt.target_contended_write_task_count.fetch_add(1, std::sync::atomic::Ordering::SeqCst)
             }
         }
         //let a = Task::clone_in_queue(this);
@@ -1340,7 +1340,7 @@ impl ScheduleStage {
     ) {
         // do par()-ly?
 
-        ee.reindex_with_address_book(ast);
+        //ee.reindex_with_address_book(ast);
         assert!(ee.is_reindexed());
 
         ee.task.record_commit_time(*commit_clock);

From d3cca29f4e5bbb4aece61f0ab8c406f9082a10a6 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 17 Sep 2022 14:04:37 +0900
Subject: [PATCH 2764/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index d68e8f0100c8f7..d3d1d2886127e0 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -104,7 +104,7 @@ impl ExecutionEnvironment {
                 });
 
             if should_remove && lock_attempt.requested_usage == RequestedUsage::Writable {
-                lock_attempt.target_contended_write_task_count().fetch_sub(1, std::sync::atomic::Ordering::SeqCst););
+                lock_attempt.target_contended_write_task_count().fetch_sub(1, std::sync::atomic::Ordering::SeqCst);
             }
         }
     }

From 8bf3fbc161d3808776f1f143248b73ab9ff650a5 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 17 Sep 2022 14:04:51 +0900
Subject: [PATCH 2765/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index d3d1d2886127e0..b5a2f558d56fa0 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -747,7 +747,7 @@ impl Task {
             lock_attempt.target_contended_unique_weights().insert_task(this.unique_weight, Task::clone_in_queue(this));
 
             if lock_attempt.requested_usage == RequestedUsage::Writable {
-                lock_attempt.target_contended_write_task_count.fetch_add(1, std::sync::atomic::Ordering::SeqCst)
+                lock_attempt.target_contended_write_task_count().fetch_add(1, std::sync::atomic::Ordering::SeqCst)
             }
         }
         //let a = Task::clone_in_queue(this);

From 85436e8099f27bf6fdd5418d70f735bc13c3a837 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 17 Sep 2022 14:05:03 +0900
Subject: [PATCH 2766/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index b5a2f558d56fa0..e63a8b54e55af4 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -173,7 +173,7 @@ impl LockAttempt {
         &self.target.0.1
     }
 
-    pub fn target_contended_write_task_count(&self) -> &AtomicUsize {
+    pub fn target_contended_write_task_count(&self) -> &std::sync::atomic::AtomicUsize {
         &self.target.0.2
     }
 }

From a56624fb118878d116e89b1e8b7839bc5daecadf Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 17 Sep 2022 14:05:16 +0900
Subject: [PATCH 2767/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index e63a8b54e55af4..0beab3eca85277 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -19,7 +19,7 @@ type PageRcInner<T> = std::rc::Rc<T>;
 unsafe impl Send for PageRc {}
 */
 
-type PageRcInner = triomphe::Arc<(std::cell::RefCell<Page>, TaskIds, AtomicUsize)>;
+type PageRcInner = triomphe::Arc<(std::cell::RefCell<Page>, TaskIds, std::sync::atomic::AtomicUsize)>;
 
 #[derive(Debug, Clone)]
 pub struct PageRc(PageRcInner);

From d76baddb122295af22d2eb24d3d18b7afff3465e Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 17 Sep 2022 14:05:26 +0900
Subject: [PATCH 2768/3199] save

---
 scheduler/src/lib.rs | 1 +
 1 file changed, 1 insertion(+)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 0beab3eca85277..fd0654b59a07d0 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -502,6 +502,7 @@ impl Preloader {
             PageRc(PageRcInner::new((
                 core::cell::RefCell::new(Page::new(&address, Usage::unused())),
                 Default::default(),
+                Default::default(),
             )))
         }))
     }

From be8423b395a0750f710c72d1383417f0094c26d5 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 17 Sep 2022 14:05:42 +0900
Subject: [PATCH 2769/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index fd0654b59a07d0..5173c7aaad81bb 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -748,7 +748,7 @@ impl Task {
             lock_attempt.target_contended_unique_weights().insert_task(this.unique_weight, Task::clone_in_queue(this));
 
             if lock_attempt.requested_usage == RequestedUsage::Writable {
-                lock_attempt.target_contended_write_task_count().fetch_add(1, std::sync::atomic::Ordering::SeqCst)
+                lock_attempt.target_contended_write_task_count().fetch_add(1, std::sync::atomic::Ordering::SeqCst);
             }
         }
         //let a = Task::clone_in_queue(this);

From 0d51b4eaab6f2d9ba2b36d3f152997cc086686a4 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 17 Sep 2022 14:06:02 +0900
Subject: [PATCH 2770/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 5173c7aaad81bb..cbb972ec0bca1a 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -52,7 +52,7 @@ impl ExecutionEnvironment {
     //}
     //
     #[inline(never)]
-    fn reindex_with_address_book(&mut self) {
+    pub fn reindex_with_address_book(&mut self) {
         assert!(!self.is_reindexed());
         self.is_reindexed = true;
 

From eeaf5959ba94ad10c36e162c5bef8161e0ac8461 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 17 Sep 2022 14:53:16 +0900
Subject: [PATCH 2771/3199] save

---
 scheduler/src/lib.rs | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index cbb972ec0bca1a..cabb3c74f44f79 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -328,9 +328,11 @@ impl AddressBook {
         unique_weight: &UniqueWeight,
         attempt: &mut LockAttempt,
     ) -> CU {
-        let strictly_lockable_for_replay = if attempt.target_contended_unique_weights().task_ids.is_empty() {
+        let tcuw = attempt.target_contended_unique_weights().task_ids.back();
+
+        let strictly_lockable_for_replay = if tcuw.is_none() {
             true
-        } else if attempt.target_contended_unique_weights().task_ids.back().unwrap().key() == unique_weight {
+        } else if tcuw.unwrap().key() == unique_weight {
             true
         } else if attempt.requested_usage == RequestedUsage::Readonly && attempt.target_contended_write_task_count().load(std::sync::atomic::Ordering::SeqCst) == 0 {
             true

From 5b60903568ec6036cac42168746a2f90afd823ce Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 17 Sep 2022 14:54:50 +0900
Subject: [PATCH 2772/3199] save

---
 scheduler/src/lib.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index cabb3c74f44f79..5912779e5ba9d1 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -328,11 +328,11 @@ impl AddressBook {
         unique_weight: &UniqueWeight,
         attempt: &mut LockAttempt,
     ) -> CU {
-        let tcuw = attempt.target_contended_unique_weights().task_ids.back();
+        let tcuw = attempt.target_contended_unique_weights().task_ids.back().map(|j| j.key()).copied();
 
         let strictly_lockable_for_replay = if tcuw.is_none() {
             true
-        } else if tcuw.unwrap().key() == unique_weight {
+        } else if tcuw.unwrap() == unique_weight {
             true
         } else if attempt.requested_usage == RequestedUsage::Readonly && attempt.target_contended_write_task_count().load(std::sync::atomic::Ordering::SeqCst) == 0 {
             true

From 9a94df8ad62ca1d36c43e6559bd83f93629477e4 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 17 Sep 2022 14:55:05 +0900
Subject: [PATCH 2773/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 5912779e5ba9d1..f2eb86c927add3 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -332,7 +332,7 @@ impl AddressBook {
 
         let strictly_lockable_for_replay = if tcuw.is_none() {
             true
-        } else if tcuw.unwrap() == unique_weight {
+        } else if tcuw.unwrap() == *unique_weight {
             true
         } else if attempt.requested_usage == RequestedUsage::Readonly && attempt.target_contended_write_task_count().load(std::sync::atomic::Ordering::SeqCst) == 0 {
             true

From ae7fc3c0261139c82f349d90b800a03aea8d8d99 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 17 Sep 2022 14:55:28 +0900
Subject: [PATCH 2774/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index f2eb86c927add3..b28992db8826e9 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -328,7 +328,7 @@ impl AddressBook {
         unique_weight: &UniqueWeight,
         attempt: &mut LockAttempt,
     ) -> CU {
-        let tcuw = attempt.target_contended_unique_weights().task_ids.back().map(|j| j.key()).copied();
+        let tcuw = attempt.target_contended_unique_weights().task_ids.back().map(|j| *j.key());
 
         let strictly_lockable_for_replay = if tcuw.is_none() {
             true

From b8767a2ee28e52936d7dd3cfb5864edb0993fd2d Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 17 Sep 2022 14:58:35 +0900
Subject: [PATCH 2775/3199] save

---
 core/src/replay_stage.rs | 1 +
 1 file changed, 1 insertion(+)

diff --git a/core/src/replay_stage.rs b/core/src/replay_stage.rs
index b181287a512237..9106f1843639ad 100644
--- a/core/src/replay_stage.rs
+++ b/core/src/replay_stage.rs
@@ -2472,6 +2472,7 @@ impl ReplayStage {
 
             assert_eq!(bank_slot, bank.slot());
             if bank.is_complete() {
+                info!("waiting for completed bank: slot: {}",bank.slot());
                 let r = bank.wait_for_scheduler(false);
                 if let Err(err) = r {
                     // Error means the slot needs to be marked as dead

From f4f25bfdf8c90a64b0e452479a29f3fd1aab56b1 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 17 Sep 2022 15:38:59 +0900
Subject: [PATCH 2776/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index b28992db8826e9..aba7631784341b 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1711,7 +1711,7 @@ impl ScheduleStage {
                     }
                 }
             }
-            assert!(!select_skipped || executing_queue_count > 0);
+            //assert!(!select_skipped || executing_queue_count > 0);
         }
         drop(to_next_stage);
         drop(ee_sender);

From 98582fe188cddb6aaf726c7d86ae7e078b9a66b8 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 17 Sep 2022 15:40:28 +0900
Subject: [PATCH 2777/3199] save

---
 scheduler/src/lib.rs | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index aba7631784341b..afdfb8a27f6ced 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1711,7 +1711,9 @@ impl ScheduleStage {
                     }
                 }
             }
-            //assert!(!select_skipped || executing_queue_count > 0);
+            if !(!select_skipped || executing_queue_count > 0) {
+                warn!("very odd {} {}", select_skipped, executing_queue_count);
+            }
         }
         drop(to_next_stage);
         drop(ee_sender);

From 957ee4bd19ab3067ad65641710d0e80526c23c22 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 17 Sep 2022 16:20:28 +0900
Subject: [PATCH 2778/3199] save

---
 runtime/src/bank.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 4c0daac69dab8d..54ea4409be34e1 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1386,7 +1386,7 @@ impl Scheduler {
 
                 ee.execution_result = Some(tx_result);
 
-                ee.reindex_with_address_book();
+                //ee.reindex_with_address_book();
                 processed_ee_sender.send(solana_scheduler::UnlockablePayload(ee)).unwrap();
             }
             todo!();

From 3710fae5392fd02c24684bb844725ef795477b2c Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 17 Sep 2022 16:20:55 +0900
Subject: [PATCH 2779/3199] save

---
 scheduler/src/lib.rs | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index afdfb8a27f6ced..a2536b18b146f0 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1343,7 +1343,7 @@ impl ScheduleStage {
     ) {
         // do par()-ly?
 
-        //ee.reindex_with_address_book(ast);
+        ee.reindex_with_address_book(ast);
         assert!(ee.is_reindexed());
 
         ee.task.record_commit_time(*commit_clock);
@@ -1711,9 +1711,7 @@ impl ScheduleStage {
                     }
                 }
             }
-            if !(!select_skipped || executing_queue_count > 0) {
-                warn!("very odd {} {}", select_skipped, executing_queue_count);
-            }
+            assert!(!select_skipped || executing_queue_count > 0);
         }
         drop(to_next_stage);
         drop(ee_sender);

From bf22787fca6f61ba77321f8b3b948b84ac450139 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 17 Sep 2022 16:21:07 +0900
Subject: [PATCH 2780/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index a2536b18b146f0..93876ff09ef750 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1343,7 +1343,7 @@ impl ScheduleStage {
     ) {
         // do par()-ly?
 
-        ee.reindex_with_address_book(ast);
+        ee.reindex_with_address_book();
         assert!(ee.is_reindexed());
 
         ee.task.record_commit_time(*commit_clock);

From bdb3d140e9a0b9e63cc86ef0ce2dd7f68cdefd7b Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 17 Sep 2022 20:58:26 +0900
Subject: [PATCH 2781/3199] save

---
 scheduler/src/lib.rs | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 93876ff09ef750..7b1b084e95ae73 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1593,11 +1593,9 @@ impl ScheduleStage {
                     if let Some(ee) = maybe_ee {
                         executing_queue_count = executing_queue_count.checked_add(1).unwrap();
                         to_execute_substage.send(ExecutablePayload(ee)).unwrap();
-                        debug!("schedule_once id_{:016x} [C] ch(prev: {}, exec: {}|{}), r: {}, u/c: {}/{}, (imm+provi)/max: ({}+{})/{} s: {} l(s+f): {}+{}", random_id, (if from_disconnected { "-".to_string() } else { format!("{}", from_prev.len()) }), to_execute_substage.len(), from_exec.len(), runnable_queue.task_count(), address_book.uncontended_task_ids.len(), contended_count, executing_queue_count, provisioning_tracker_count, max_executing_queue_count, address_book.stuck_tasks.len(), processed_count, failed_lock_count);
-                    } else {
-                        debug!("schedule_once id_{:016x} [C] ch(prev: {}, exec: {}|{}), r: {}, u/c: {}/{}, (imm+provi)/max: ({}+{})/{} s: {} l(s+f): {}+{}", random_id, (if from_disconnected { "-".to_string() } else { format!("{}", from_prev.len()) }), to_execute_substage.len(), from_exec.len(), runnable_queue.task_count(), address_book.uncontended_task_ids.len(), contended_count, executing_queue_count, provisioning_tracker_count, max_executing_queue_count, address_book.stuck_tasks.len(), processed_count, failed_lock_count);
-                        break;
                     }
+                    debug!("schedule_once id_{:016x} [C] ch(prev: {}, exec: {}|{}), r: {}, u/c: {}/{}, (imm+provi)/max: ({}+{})/{} s: {} l(s+f): {}+{}", random_id, (if from_disconnected { "-".to_string() } else { format!("{}", from_prev.len()) }), to_execute_substage.len(), from_exec.len(), runnable_queue.task_count(), address_book.uncontended_task_ids.len(), contended_count, executing_queue_count, provisioning_tracker_count, max_executing_queue_count, address_book.stuck_tasks.len(), processed_count, failed_lock_count);
+
                     if no_aggresive_contended && !from_exec.is_empty() {
                         trace!("abort aggressive contended queue processing due to non-empty from_exec");
                         break;

From b9a4d09594d772a148164aabee07388dadcd076a Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 17 Sep 2022 21:00:25 +0900
Subject: [PATCH 2782/3199] save

---
 scheduler/src/lib.rs | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 7b1b084e95ae73..d5d3c20ecd6088 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1633,11 +1633,9 @@ impl ScheduleStage {
                     if let Some(ee) = maybe_ee {
                         executing_queue_count = executing_queue_count.checked_add(1).unwrap();
                         to_execute_substage.send(ExecutablePayload(ee)).unwrap();
-                        debug!("schedule_once id_{:016x} [R] ch(prev: {}, exec: {}|{}), r: {}, u/c: {}/{}, (imm+provi)/max: ({}+{})/{} s: {} l(s+f): {}+{}", random_id, (if from_disconnected { "-".to_string() } else { format!("{}", from_prev.len()) }), to_execute_substage.len(), from_exec.len(), runnable_queue.task_count(), address_book.uncontended_task_ids.len(), contended_count, executing_queue_count, provisioning_tracker_count, max_executing_queue_count, address_book.stuck_tasks.len(), processed_count, failed_lock_count);
-                    } else {
-                        debug!("schedule_once id_{:016x} [R] ch(prev: {}, exec: {}|{}), r: {}, u/c: {}/{}, (imm+provi)/max: ({}+{})/{} s: {} l(s+f): {}+{}", random_id, (if from_disconnected { "-".to_string() } else { format!("{}", from_prev.len()) }), to_execute_substage.len(), from_exec.len(), runnable_queue.task_count(), address_book.uncontended_task_ids.len(), contended_count, executing_queue_count, provisioning_tracker_count, max_executing_queue_count, address_book.stuck_tasks.len(), processed_count, failed_lock_count);
-                        break;
                     }
+                    debug!("schedule_once id_{:016x} [R] ch(prev: {}, exec: {}|{}), r: {}, u/c: {}/{}, (imm+provi)/max: ({}+{})/{} s: {} l(s+f): {}+{}", random_id, (if from_disconnected { "-".to_string() } else { format!("{}", from_prev.len()) }), to_execute_substage.len(), from_exec.len(), runnable_queue.task_count(), address_book.uncontended_task_ids.len(), contended_count, executing_queue_count, provisioning_tracker_count, max_executing_queue_count, address_book.stuck_tasks.len(), processed_count, failed_lock_count);
+
                     if !from_exec.is_empty() {
                         trace!("abort aggressive runnable queue processing due to non-empty from_exec");
                         break;

From eab9db8927cb06cb9844f1d4ff118b6d5f66641f Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 17 Sep 2022 21:14:02 +0900
Subject: [PATCH 2783/3199] save

---
 ledger-tool/src/main.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ledger-tool/src/main.rs b/ledger-tool/src/main.rs
index 3034bcab690969..a1988e14701995 100644
--- a/ledger-tool/src/main.rs
+++ b/ledger-tool/src/main.rs
@@ -440,7 +440,7 @@ fn output_slot(
                                 //    lock_attempt.contended_unique_weights().insert_task(weight, solana_scheduler::TaskInQueue::clone(&t));
                                 //}
 
-                                muxed_sender.send(solana_scheduler::SchedulablePayload(t)).unwrap();
+                                muxed_sender.send(solana_scheduler::SchedulablePayloadolana_scheduler::Flushable::Payload(t))).unwrap();
                                 depth.fetch_add(1, Ordering::Relaxed);
                                 weight -= 1;
                             }

From 557d0c729694e93f9cabf2d78547b423039ff62c Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 17 Sep 2022 21:14:09 +0900
Subject: [PATCH 2784/3199] save

---
 ledger-tool/src/main.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ledger-tool/src/main.rs b/ledger-tool/src/main.rs
index a1988e14701995..9ca221ba92c906 100644
--- a/ledger-tool/src/main.rs
+++ b/ledger-tool/src/main.rs
@@ -440,7 +440,7 @@ fn output_slot(
                                 //    lock_attempt.contended_unique_weights().insert_task(weight, solana_scheduler::TaskInQueue::clone(&t));
                                 //}
 
-                                muxed_sender.send(solana_scheduler::SchedulablePayloadolana_scheduler::Flushable::Payload(t))).unwrap();
+                                muxed_sender.send(solana_scheduler::SchedulablePayloadolana_scheduler::Flushable::Payload(t)).unwrap();
                                 depth.fetch_add(1, Ordering::Relaxed);
                                 weight -= 1;
                             }

From a1329e48ecd07d23031d95ac70269c00e26d6b61 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 17 Sep 2022 21:14:45 +0900
Subject: [PATCH 2785/3199] save

---
 ledger-tool/src/main.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ledger-tool/src/main.rs b/ledger-tool/src/main.rs
index 9ca221ba92c906..2915f67e9e77d8 100644
--- a/ledger-tool/src/main.rs
+++ b/ledger-tool/src/main.rs
@@ -440,7 +440,7 @@ fn output_slot(
                                 //    lock_attempt.contended_unique_weights().insert_task(weight, solana_scheduler::TaskInQueue::clone(&t));
                                 //}
 
-                                muxed_sender.send(solana_scheduler::SchedulablePayloadolana_scheduler::Flushable::Payload(t)).unwrap();
+                                muxed_sender.send(solana_scheduler::SchedulablePayload(solana_scheduler::Flushable::Payload(t))).unwrap();
                                 depth.fetch_add(1, Ordering::Relaxed);
                                 weight -= 1;
                             }

From c8f233968e0dc78e887c9ba53d41d574534c8751 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 17 Sep 2022 21:44:06 +0900
Subject: [PATCH 2786/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index d5d3c20ecd6088..517d55dcde0a3b 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1613,7 +1613,7 @@ impl ScheduleStage {
                     }
                 }
                 let mut selection = TaskSelection::OnlyFromRunnable;
-                while !runnable_finished && selection.should_proceed() && executing_queue_count + provisioning_tracker_count < max_executing_queue_count {
+                while !runnable_queue.has_no_task() && selection.should_proceed() && executing_queue_count + provisioning_tracker_count < max_executing_queue_count {
                     let prefer_immediate = true; //provisioning_tracker_count / 4 > executing_queue_count;
 
                     let maybe_ee = Self::schedule_next_execution(

From e7cbf77ba9ff1613c97cba878e0c6bc53132b521 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 17 Sep 2022 21:56:42 +0900
Subject: [PATCH 2787/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 517d55dcde0a3b..a64dae45a18c96 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1573,7 +1573,7 @@ impl ScheduleStage {
                 let runnable_finished = from_disconnected && runnable_queue.has_no_task();
 
                 let mut selection = TaskSelection::OnlyFromContended(if runnable_finished { usize::max_value() } else { usize::max_value() /*2*/ });
-                while contended_count > 0 && selection.should_proceed() && executing_queue_count + provisioning_tracker_count < max_executing_queue_count {
+                while !address_book.uncontended_task_ids.is_empty() && selection.should_proceed() && executing_queue_count + provisioning_tracker_count < max_executing_queue_count {
                     let prefer_immediate = true; //provisioning_tracker_count / 4 > executing_queue_count;
 
                     let maybe_ee = Self::schedule_next_execution(

From 66114fc0a1bec676099b32b7e5499456a33ec3f0 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 17 Sep 2022 23:34:54 +0900
Subject: [PATCH 2788/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index a64dae45a18c96..ee9b9671b0cc48 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1573,7 +1573,7 @@ impl ScheduleStage {
                 let runnable_finished = from_disconnected && runnable_queue.has_no_task();
 
                 let mut selection = TaskSelection::OnlyFromContended(if runnable_finished { usize::max_value() } else { usize::max_value() /*2*/ });
-                while !address_book.uncontended_task_ids.is_empty() && selection.should_proceed() && executing_queue_count + provisioning_tracker_count < max_executing_queue_count {
+                while !address_book.uncontended_task_ids.is_empty() && selection.should_proceed() {
                     let prefer_immediate = true; //provisioning_tracker_count / 4 > executing_queue_count;
 
                     let maybe_ee = Self::schedule_next_execution(

From efade948c4e30ced5965a348fe2d4e8fd2bd3a74 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 18 Sep 2022 14:28:04 +0900
Subject: [PATCH 2789/3199] save

---
 ledger/src/blockstore_processor.rs | 76 +-----------------------------
 scheduler/src/lib.rs               |  9 ++--
 2 files changed, 5 insertions(+), 80 deletions(-)

diff --git a/ledger/src/blockstore_processor.rs b/ledger/src/blockstore_processor.rs
index 68a6d76c96c23b..e644d759818a78 100644
--- a/ledger/src/blockstore_processor.rs
+++ b/ledger/src/blockstore_processor.rs
@@ -167,12 +167,6 @@ fn aggregate_total_execution_units(execute_timings: &ExecuteTimings) -> u64 {
 fn execute_batch(
     batch: &TransactionBatchWithIndexes,
     bank: &Arc<Bank>,
-    transaction_status_sender: Option<&TransactionStatusSender>,
-    replay_vote_sender: Option<&ReplayVoteSender>,
-    timings: &mut ExecuteTimings,
-    cost_capacity_meter: Arc<RwLock<BlockCostCapacityMeter>>,
-    tx_cost: u64,
-    log_messages_bytes_limit: Option<usize>,
 ) -> Result<()> {
     let TransactionBatchWithIndexes {
         batch,
@@ -190,8 +184,6 @@ fn execute_batch(
     };
     */
 
-    let pre_process_units: u64 = aggregate_total_execution_units(timings);
-
     batch.bank().schedule_and_commit_transactions(
         batch,
         bank,
@@ -304,8 +296,7 @@ fn execute_batches_internal(
     let results: Vec<Result<()>> = 
         batches
             .into_iter()
-            .enumerate()
-            .map(|(index, transaction_batch_with_indexes)| {
+            .map(|transaction_batch_with_indexes| {
                 let transaction_count = transaction_batch_with_indexes
                     .batch
                     .sanitized_transactions()
@@ -316,12 +307,6 @@ fn execute_batches_internal(
                         let result = execute_batch(
                             transaction_batch_with_indexes,
                             bank,
-                            transaction_status_sender,
-                            replay_vote_sender,
-                            &mut timings,
-                            cost_capacity_meter.clone(),
-                            tx_costs[index],
-                            log_messages_bytes_limit,
                         );
                         if let Some(entry_callback) = entry_callback {
                             entry_callback(bank);
@@ -420,68 +405,11 @@ fn execute_batches(
 
     let mut minimal_tx_cost = u64::MAX;
     let mut total_cost: u64 = 0;
-    let mut total_cost_without_bpf: u64 = 0;
-    // Allowing collect here, since it also computes the minimal tx cost, and aggregate cost.
-    // These two values are later used for checking if the tx_costs vector needs to be iterated over.
-    // The collection is a pair of (full cost, cost without estimated-bpf-code-costs).
-    #[allow(clippy::needless_collect)]
-    let tx_costs = sanitized_txs
-        .iter()
-        .map(|tx| {
-            let tx_cost = cost_model.calculate_cost(tx);
-            let cost = tx_cost.sum();
-            let cost_without_bpf = tx_cost.sum_without_bpf();
-            minimal_tx_cost = std::cmp::min(minimal_tx_cost, cost);
-            total_cost = total_cost.saturating_add(cost);
-            total_cost_without_bpf = total_cost_without_bpf.saturating_add(cost_without_bpf);
-            (cost, cost_without_bpf)
-        })
-        .collect::<Vec<_>>();
-
-    let target_batch_count = get_thread_count() as u64;
-
     let mut tx_batches: Vec<TransactionBatchWithIndexes> = vec![];
     let mut tx_batch_costs: Vec<u64> = vec![];
-    let rebatched_txs = if total_cost > target_batch_count.saturating_mul(minimal_tx_cost) {
-        let target_batch_cost = total_cost / target_batch_count;
-        let mut batch_cost: u64 = 0;
-        let mut batch_cost_without_bpf: u64 = 0;
-        let mut slice_start = 0;
-        tx_costs
-            .into_iter()
-            .enumerate()
-            .for_each(|(index, cost_pair)| {
-                let next_index = index + 1;
-                batch_cost = batch_cost.saturating_add(cost_pair.0);
-                batch_cost_without_bpf = batch_cost_without_bpf.saturating_add(cost_pair.1);
-                if batch_cost >= target_batch_cost || next_index == sanitized_txs.len() {
-                    let tx_batch = rebatch_transactions(
-                        &lock_results,
-                        bank,
-                        &sanitized_txs,
-                        slice_start,
-                        index,
-                        &transaction_indexes,
-                    );
-                    slice_start = next_index;
-                    tx_batches.push(tx_batch);
-                    tx_batch_costs.push(batch_cost_without_bpf);
-                    batch_cost = 0;
-                    batch_cost_without_bpf = 0;
-                }
-            });
-        &tx_batches[..]
-    } else {
-        // Ensure that the total cost attributed to this batch is essentially correct
-        // batches.is_empty() check at top of function ensures that n > 0 for following divide
-        let n = batches.len();
-        tx_batch_costs = vec![total_cost_without_bpf / (n as u64); n];
-        batches
-    };
-
     let execute_batches_internal_metrics = execute_batches_internal(
         bank,
-        rebatched_txs,
+        batches,
         entry_callback,
         transaction_status_sender,
         replay_vote_sender,
diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index ee9b9671b0cc48..10c4caba1a407d 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -72,28 +72,25 @@ impl ExecutionEnvironment {
                     let mut removed = false;
                     let mut task = task_cursor.value();
                     //task.trace_timestamps("in_exec(initial list)");
+                    assert!(!task.already_finished());
                     while !task.currently_contended() {
                         if task_cursor.key() == &uq {
                             assert!(should_remove);
                             removed = task_cursor.remove();
                             assert!(removed);
                         }
-                        if task.already_finished() {
-                            task_cursor.remove();
-                        }
                         if let Some(new_cursor) = task_cursor.prev() {
                             assert!(new_cursor.key() < task_cursor.key());
                             task_cursor = new_cursor;
                             task = task_cursor.value();
+                            assert!(!task.already_finished());
                             //task.trace_timestamps("in_exec(subsequent list)");
                         } else {
                             found = false;
                             break;
                         }
                     }
-                    if should_remove && !removed {
-                        contended_unique_weights.remove_task(&uq);
-                    }
+                    assert!(!should_remove || removed);
                     found.then(|| Task::clone_in_queue(task))
                 })
                 .flatten()

From dcafe74a662e91509f4bab2013b387651aa153f7 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 18 Sep 2022 14:36:26 +0900
Subject: [PATCH 2790/3199] save

---
 ledger/src/blockstore_processor.rs | 51 ++++--------------------------
 1 file changed, 7 insertions(+), 44 deletions(-)

diff --git a/ledger/src/blockstore_processor.rs b/ledger/src/blockstore_processor.rs
index e644d759818a78..e44d7bdd4d508b 100644
--- a/ledger/src/blockstore_processor.rs
+++ b/ledger/src/blockstore_processor.rs
@@ -288,56 +288,19 @@ fn execute_batches_internal(
     log_messages_bytes_limit: Option<usize>,
 ) -> Result<ExecuteBatchesInternalMetrics> {
     assert!(!batches.is_empty());
-    inc_new_counter_debug!("bank-par_execute_entries-count", batches.len());
-    let execution_timings_per_thread: Mutex<HashMap<usize, ThreadExecuteTimings>> =
-        Mutex::new(HashMap::new());
 
     let mut execute_batches_elapsed = Measure::start("execute_batches_elapsed");
     let results: Vec<Result<()>> = 
         batches
             .into_iter()
             .map(|transaction_batch_with_indexes| {
-                let transaction_count = transaction_batch_with_indexes
-                    .batch
-                    .sanitized_transactions()
-                    .len() as u64;
-                let mut timings = ExecuteTimings::default();
-                let (result, execute_batches_time): (Result<()>, Measure) = measure!(
-                    {
-                        let result = execute_batch(
-                            transaction_batch_with_indexes,
-                            bank,
-                        );
-                        if let Some(entry_callback) = entry_callback {
-                            entry_callback(bank);
-                        }
-                        result
-                    },
-                    "execute_batch",
+                let result = execute_batch(
+                    transaction_batch_with_indexes,
+                    bank,
                 );
-
-                let thread_index = 0;// PAR_THREAD_POOL.current_thread_index().unwrap();
-                execution_timings_per_thread
-                    .lock()
-                    .unwrap()
-                    .entry(thread_index)
-                    .and_modify(|thread_execution_time| {
-                        let ThreadExecuteTimings {
-                            total_thread_us,
-                            total_transactions_executed,
-                            execute_timings: total_thread_execute_timings,
-                        } = thread_execution_time;
-                        *total_thread_us += execute_batches_time.as_us();
-                        *total_transactions_executed += transaction_count;
-                        total_thread_execute_timings
-                            .saturating_add_in_place(ExecuteTimingType::TotalBatchesLen, 1);
-                        total_thread_execute_timings.accumulate(&timings);
-                    })
-                    .or_insert(ThreadExecuteTimings {
-                        total_thread_us: execute_batches_time.as_us(),
-                        total_transactions_executed: transaction_count,
-                        execute_timings: timings,
-                    });
+                if let Some(entry_callback) = entry_callback {
+                    entry_callback(bank);
+                }
                 result
             })
             .collect()
@@ -347,7 +310,7 @@ fn execute_batches_internal(
     first_err(&results)?;
 
     Ok(ExecuteBatchesInternalMetrics {
-        execution_timings_per_thread: execution_timings_per_thread.into_inner().unwrap(),
+        execution_timings_per_thread: Default::default(),
         total_batches_len: batches.len() as u64,
         execute_batches_us: execute_batches_elapsed.as_us(),
     })

From b8f2c00a7929032082febfdd9046d75cfae78b73 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 18 Sep 2022 14:38:43 +0900
Subject: [PATCH 2791/3199] save

---
 ledger/src/blockstore_processor.rs | 29 +++++++++++------------------
 1 file changed, 11 insertions(+), 18 deletions(-)

diff --git a/ledger/src/blockstore_processor.rs b/ledger/src/blockstore_processor.rs
index e44d7bdd4d508b..13b3ddef2fd1e5 100644
--- a/ledger/src/blockstore_processor.rs
+++ b/ledger/src/blockstore_processor.rs
@@ -174,7 +174,7 @@ fn execute_batch(
     } = batch;
     //let record_token_balances = transaction_status_sender.is_some();
 
-    let mut mint_decimals: HashMap<Pubkey, u8> = HashMap::new();
+    //let mut mint_decimals: HashMap<Pubkey, u8> = HashMap::new();
 
     /*
     let pre_token_balances = if record_token_balances {
@@ -290,25 +290,18 @@ fn execute_batches_internal(
     assert!(!batches.is_empty());
 
     let mut execute_batches_elapsed = Measure::start("execute_batches_elapsed");
-    let results: Vec<Result<()>> = 
-        batches
-            .into_iter()
-            .map(|transaction_batch_with_indexes| {
-                let result = execute_batch(
-                    transaction_batch_with_indexes,
-                    bank,
-                );
-                if let Some(entry_callback) = entry_callback {
-                    entry_callback(bank);
-                }
-                result
-            })
-            .collect()
-    ;
+    for batch in batches {
+        let result = execute_batch(
+            batch,
+            bank,
+        );
+        if let Some(entry_callback) = entry_callback {
+            entry_callback(bank);
+        }
+        result?
+    }
     execute_batches_elapsed.stop();
 
-    first_err(&results)?;
-
     Ok(ExecuteBatchesInternalMetrics {
         execution_timings_per_thread: Default::default(),
         total_batches_len: batches.len() as u64,

From 7e1f8886fec7848f309394f2726284b57cee9506 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 18 Sep 2022 14:51:41 +0900
Subject: [PATCH 2792/3199] save

---
 metrics/src/datapoint.rs | 74 ++++++++++++++++++++++++++++++++++++++++
 1 file changed, 74 insertions(+)

diff --git a/metrics/src/datapoint.rs b/metrics/src/datapoint.rs
index 9fcd9ea215ad5c..088f6d3c289ea0 100644
--- a/metrics/src/datapoint.rs
+++ b/metrics/src/datapoint.rs
@@ -150,6 +150,56 @@ macro_rules! create_datapoint {
     };
 }
 
+#[macro_export]
+macro_rules! create_datapoint_at {
+    (@field $point:ident $name:expr, $string:expr, String) => {
+        $point.add_field_str($name, &$string);
+    };
+    (@field $point:ident $name:expr, $value:expr, i64) => {
+        $point.add_field_i64($name, $value as i64);
+    };
+    (@field $point:ident $name:expr, $value:expr, f64) => {
+        $point.add_field_f64($name, $value as f64);
+    };
+    (@field $point:ident $name:expr, $value:expr, bool) => {
+        $point.add_field_bool($name, $value as bool);
+    };
+    (@tag $point:ident $tag_name:expr, $tag_value:expr) => {
+        $point.add_tag($tag_name, &$tag_value);
+    };
+
+    (@fields $point:ident) => {};
+
+    // process tags
+    (@fields $point:ident $tag_name:expr => $tag_value:expr, $($rest:tt)*) => {
+        $crate::create_datapoint!(@tag $point $tag_name, $tag_value);
+        $crate::create_datapoint!(@fields $point $($rest)*);
+    };
+    (@fields $point:ident $tag_name:expr => $tag_value:expr) => {
+        $crate::create_datapoint!(@tag $point $tag_name, $tag_value);
+    };
+
+    // process fields
+    (@fields $point:ident ($name:expr, $value:expr, $type:ident) , $($rest:tt)*) => {
+        $crate::create_datapoint!(@field $point $name, $value, $type);
+        $crate::create_datapoint!(@fields $point $($rest)*);
+    };
+    (@fields $point:ident ($name:expr, $value:expr, $type:ident)) => {
+        $crate::create_datapoint!(@field $point $name, $value, $type);
+    };
+
+    (@point $name:expr, $at:expr, $($fields:tt)+) => {
+        {
+            let mut point = $crate::datapoint::DataPoint::at($at, &$name);
+            $crate::create_datapoint!(@fields point $($fields)+);
+            point
+        }
+    };
+    (@point $name:expr, $at:expr) => {
+        $crate::datapoint::DataPoint::at($at, &$name)
+    };
+}
+
 #[macro_export]
 macro_rules! datapoint {
     ($level:expr, $name:expr) => {
@@ -163,6 +213,20 @@ macro_rules! datapoint {
         }
     };
 }
+
+#[macro_export]
+macro_rules! datapoint_at {
+    ($at:expr, $level:expr, $name:expr) => {
+        if log::log_enabled!($level) {
+            $crate::submit($crate::create_datapoint_at!(@point $name, $at), $level);
+        }
+    };
+    ($at:expr, $level:expr, $name:expr, $($fields:tt)+) => {
+        if log::log_enabled!($level) {
+            $crate::submit($crate::create_datapoint_at!(@point $name, $at, $($fields)+), $level);
+        }
+    };
+}
 #[macro_export]
 macro_rules! datapoint_error {
     ($name:expr) => {
@@ -193,6 +257,16 @@ macro_rules! datapoint_info {
     };
 }
 
+#[macro_export]
+macro_rules! datapoint_info_at {
+    ($at:expr, $name:expr) => {
+        $crate::datapoint_at!(log::Level::Info, $at, $name);
+    };
+    ($at:expr, $name:expr, $($fields:tt)+) => {
+        $crate::datapoint_at!(log::Level::Info, $at, $name, $($fields)+);
+    };
+}
+
 #[macro_export]
 macro_rules! datapoint_debug {
     ($name:expr) => {

From d85bff566a88ee40fe359c09101fdb9c5e25bb27 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 18 Sep 2022 14:52:14 +0900
Subject: [PATCH 2793/3199] save

---
 runtime/src/bank.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 54ea4409be34e1..2a81e1845b7cf5 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1370,7 +1370,7 @@ impl Scheduler {
                     wall_time.stop();
                     let duration_with_overhead = wall_time.as_us();
 
-                    datapoint_info!(
+                    datapoint_info_at!(
                         "individual_tx_stats",
                         ("slot", slot, i64),
                         ("index", transaction_index, i64),

From 8798f587824771932a54b88d29830caebd0542cb Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 18 Sep 2022 14:52:55 +0900
Subject: [PATCH 2794/3199] save

---
 runtime/src/bank.rs | 1 +
 1 file changed, 1 insertion(+)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 2a81e1845b7cf5..cf15630e04be91 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1371,6 +1371,7 @@ impl Scheduler {
                     let duration_with_overhead = wall_time.as_us();
 
                     datapoint_info_at!(
+                        3,
                         "individual_tx_stats",
                         ("slot", slot, i64),
                         ("index", transaction_index, i64),

From e372a5594e38abf7d46d538b965332d6ae44c43d Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 18 Sep 2022 14:53:30 +0900
Subject: [PATCH 2795/3199] save

---
 runtime/src/bank.rs | 9 ---------
 1 file changed, 9 deletions(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index cf15630e04be91..861fc7d4780920 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1373,15 +1373,6 @@ impl Scheduler {
                     datapoint_info_at!(
                         3,
                         "individual_tx_stats",
-                        ("slot", slot, i64),
-                        ("index", transaction_index, i64),
-                        ("thread", current_thread_name, String),
-                        ("signature", &sig, String),
-                        ("account_locks_in_json", "{}", String),
-                        ("status", status_str.unwrap(), String),
-                        ("duration", duration_with_overhead, i64),
-                        ("cpu_duration", cpu_time.elapsed().as_micros(), i64),
-                        ("compute_units", ee.cu, i64),
                     );
                 }
 

From b5869968ebfa1f914ec23045743d8a9047a89b64 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 18 Sep 2022 14:53:38 +0900
Subject: [PATCH 2796/3199] save

---
 runtime/src/bank.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 861fc7d4780920..ea0adc45a67211 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1372,7 +1372,7 @@ impl Scheduler {
 
                     datapoint_info_at!(
                         3,
-                        "individual_tx_stats",
+                        "individual_tx_stats"
                     );
                 }
 

From c92235e19f05d4d059f95ca61aad4136c6e1e470 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 18 Sep 2022 14:53:57 +0900
Subject: [PATCH 2797/3199] save

---
 runtime/src/bank.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index ea0adc45a67211..486b0c038be174 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1371,7 +1371,7 @@ impl Scheduler {
                     let duration_with_overhead = wall_time.as_us();
 
                     datapoint_info_at!(
-                        3,
+                        vec![],
                         "individual_tx_stats"
                     );
                 }

From 10fa3aae3894c793446aa7421efe021c7caa7d37 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 18 Sep 2022 14:54:45 +0900
Subject: [PATCH 2798/3199] save

---
 metrics/src/datapoint.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/metrics/src/datapoint.rs b/metrics/src/datapoint.rs
index 088f6d3c289ea0..a2e1efa646c9c6 100644
--- a/metrics/src/datapoint.rs
+++ b/metrics/src/datapoint.rs
@@ -216,12 +216,12 @@ macro_rules! datapoint {
 
 #[macro_export]
 macro_rules! datapoint_at {
-    ($at:expr, $level:expr, $name:expr) => {
+    ($level:expr, $at:expr, $name:expr) => {
         if log::log_enabled!($level) {
             $crate::submit($crate::create_datapoint_at!(@point $name, $at), $level);
         }
     };
-    ($at:expr, $level:expr, $name:expr, $($fields:tt)+) => {
+    ($level:expr, $at:expr, $name:expr, $($fields:tt)+) => {
         if log::log_enabled!($level) {
             $crate::submit($crate::create_datapoint_at!(@point $name, $at, $($fields)+), $level);
         }

From cf332cc5ba528221a9981d2d4f8c16406ea9f62e Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 18 Sep 2022 14:55:31 +0900
Subject: [PATCH 2799/3199] save

---
 metrics/src/datapoint.rs | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/metrics/src/datapoint.rs b/metrics/src/datapoint.rs
index a2e1efa646c9c6..a743a2f7c87a38 100644
--- a/metrics/src/datapoint.rs
+++ b/metrics/src/datapoint.rs
@@ -60,6 +60,15 @@ impl DataPoint {
         }
     }
 
+    pub fn at(name: &'static str, timestamp: SystemTime) -> Self {
+        DataPoint {
+            name,
+            timestamp,
+            tags: vec![],
+            fields: vec![],
+        }
+    }
+
     pub fn add_tag(&mut self, name: &'static str, value: &str) -> &mut Self {
         self.tags.push((name, value.to_string()));
         self

From 451a8717584a280e5757bf07f9c3bb20c834e358 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 18 Sep 2022 14:56:20 +0900
Subject: [PATCH 2800/3199] save

---
 runtime/src/bank.rs | 13 +++++++++++--
 1 file changed, 11 insertions(+), 2 deletions(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 486b0c038be174..3b0d6363c17f77 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1371,8 +1371,17 @@ impl Scheduler {
                     let duration_with_overhead = wall_time.as_us();
 
                     datapoint_info_at!(
-                        vec![],
-                        "individual_tx_stats"
+                        std::time::SystemTimw.now(),
+                        "individual_tx_stats",
+                        ("slot", slot, i64),
+                        ("index", transaction_index, i64),
+                        ("thread", current_thread_name, String),
+                        ("signature", &sig, String),
+                        ("account_locks_in_json", "{}", String),
+                        ("status", status_str.unwrap(), String),
+                        ("duration", duration_with_overhead, i64),
+                        ("cpu_duration", cpu_time.elapsed().as_micros(), i64),
+                        ("compute_units", ee.cu, i64),
                     );
                 }
 

From 03c80427e2517f44df077001e18874e18df45e5e Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 18 Sep 2022 14:56:42 +0900
Subject: [PATCH 2801/3199] save

---
 runtime/src/bank.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 3b0d6363c17f77..2e2cf7871ac9f9 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1371,7 +1371,7 @@ impl Scheduler {
                     let duration_with_overhead = wall_time.as_us();
 
                     datapoint_info_at!(
-                        std::time::SystemTimw.now(),
+                        std::time::SystemTime.now(),
                         "individual_tx_stats",
                         ("slot", slot, i64),
                         ("index", transaction_index, i64),

From 7ea5e300ae59fc05576ce366f27b9e521ea61d2c Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 18 Sep 2022 14:56:54 +0900
Subject: [PATCH 2802/3199] save

---
 runtime/src/bank.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 2e2cf7871ac9f9..fb3805e9a1d3a0 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1371,7 +1371,7 @@ impl Scheduler {
                     let duration_with_overhead = wall_time.as_us();
 
                     datapoint_info_at!(
-                        std::time::SystemTime.now(),
+                        std::time::SystemTime::now(),
                         "individual_tx_stats",
                         ("slot", slot, i64),
                         ("index", transaction_index, i64),

From ed3b077c62ffec713a96dad5a91bf3c20767d012 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 18 Sep 2022 14:57:24 +0900
Subject: [PATCH 2803/3199] save

---
 metrics/src/datapoint.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/metrics/src/datapoint.rs b/metrics/src/datapoint.rs
index a743a2f7c87a38..dba7fe9104239b 100644
--- a/metrics/src/datapoint.rs
+++ b/metrics/src/datapoint.rs
@@ -60,7 +60,7 @@ impl DataPoint {
         }
     }
 
-    pub fn at(name: &'static str, timestamp: SystemTime) -> Self {
+    pub fn at(timestamp: SystemTime, name: &'static str) -> Self {
         DataPoint {
             name,
             timestamp,

From e600ccc081306c2cfa5250c7e63217366f3d11cd Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 18 Sep 2022 15:03:50 +0900
Subject: [PATCH 2804/3199] save

---
 runtime/src/bank.rs | 50 ++++++++++++++++++++++-----------------------
 1 file changed, 24 insertions(+), 26 deletions(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index fb3805e9a1d3a0..4505d1f5dbfad1 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1353,38 +1353,15 @@ impl Scheduler {
                 } = tx_results;
 
                 let tx_result = fee_collection_results.into_iter().collect::<Result<_>>();
-                let status_str = if tx_result.is_ok() {
-                    let details = execution_results[0].details().expect(&format!("tx_result: {:?}", tx_result));
+                if tx_result.is_ok() {
+                    let details = execution_results[0].details().unwrap();
                     ee.cu = details.executed_units;
-                    send_metrics.then(|| format!("{:?}", details.status))
                 } else {
                     let sig = ee.task.tx.0.signature().to_string();
                     error!("found odd tx error: slot: {}, signature: {}, {:?}", slot, sig, tx_result);
-
-                    send_metrics.then(|| format!("{:?}", tx_result))
                 };
 
-                if send_metrics {
-                    let sig = ee.task.tx.0.signature().to_string();
-
-                    wall_time.stop();
-                    let duration_with_overhead = wall_time.as_us();
-
-                    datapoint_info_at!(
-                        std::time::SystemTime::now(),
-                        "individual_tx_stats",
-                        ("slot", slot, i64),
-                        ("index", transaction_index, i64),
-                        ("thread", current_thread_name, String),
-                        ("signature", &sig, String),
-                        ("account_locks_in_json", "{}", String),
-                        ("status", status_str.unwrap(), String),
-                        ("duration", duration_with_overhead, i64),
-                        ("cpu_duration", cpu_time.elapsed().as_micros(), i64),
-                        ("compute_units", ee.cu, i64),
-                    );
-                }
-
+                ee.finish_time = std::timeSystemTime::now();
                 ee.execution_result = Some(tx_result);
 
                 //ee.reindex_with_address_book();
@@ -1405,6 +1382,27 @@ impl Scheduler {
             }
 
             while let Ok(solana_scheduler::ExaminablePayload(mut ee)) = retired_ee_receiver.recv() {
+                if send_metrics {
+                    let sig = ee.task.tx.0.signature().to_string();
+
+                    wall_time.stop();
+                    let duration_with_overhead = wall_time.as_us();
+
+                    datapoint_info_at!(
+                        ee.finish_time,
+                        "individual_tx_stats",
+                        ("slot", slot, i64),
+                        ("index", transaction_index, i64),
+                        ("thread", current_thread_name, String),
+                        ("signature", &sig, String),
+                        ("account_locks_in_json", "{}", String),
+                        ("status", ee.execution_result.unwrap(), String),
+                        ("duration", duration_with_overhead, i64),
+                        ("cpu_duration", cpu_time.elapsed().as_micros(), i64),
+                        ("compute_units", ee.cu, i64),
+                    );
+                }
+
                         if ee.is_aborted() {
                             warn!(
                                 "scheduler: Unexpected validator error: {:?}, transaction: {:?}",

From 63f4562b5d6ce0b2b0423b61df26c4e46fce5d6c Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 18 Sep 2022 15:04:05 +0900
Subject: [PATCH 2805/3199] save

---
 runtime/src/bank.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 4505d1f5dbfad1..47815343e5e6b4 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1361,7 +1361,7 @@ impl Scheduler {
                     error!("found odd tx error: slot: {}, signature: {}, {:?}", slot, sig, tx_result);
                 };
 
-                ee.finish_time = std::timeSystemTime::now();
+                ee.finish_time = std::time::SystemTime::now();
                 ee.execution_result = Some(tx_result);
 
                 //ee.reindex_with_address_book();

From 0d020bcb44e03a0a66bcb3e33a172b0e22c5a404 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 18 Sep 2022 15:06:00 +0900
Subject: [PATCH 2806/3199] save

---
 runtime/src/bank.rs | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 47815343e5e6b4..6ea3b0d94f7b22 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1363,6 +1363,9 @@ impl Scheduler {
 
                 ee.finish_time = std::time::SystemTime::now();
                 ee.execution_result = Some(tx_result);
+                wall_time.stop();
+                ee.execution_us = wall_time.as_us();
+                ee.execution_cpu_us = cpu_time.elapsed().as_micros();
 
                 //ee.reindex_with_address_book();
                 processed_ee_sender.send(solana_scheduler::UnlockablePayload(ee)).unwrap();
@@ -1385,8 +1388,6 @@ impl Scheduler {
                 if send_metrics {
                     let sig = ee.task.tx.0.signature().to_string();
 
-                    wall_time.stop();
-                    let duration_with_overhead = wall_time.as_us();
 
                     datapoint_info_at!(
                         ee.finish_time,
@@ -1397,8 +1398,8 @@ impl Scheduler {
                         ("signature", &sig, String),
                         ("account_locks_in_json", "{}", String),
                         ("status", ee.execution_result.unwrap(), String),
-                        ("duration", duration_with_overhead, i64),
-                        ("cpu_duration", cpu_time.elapsed().as_micros(), i64),
+                        ("duration", ee.execution_us, i64),
+                        ("cpu_duration", ee.execution_cpu_us, i64),
                         ("compute_units", ee.cu, i64),
                     );
                 }

From e696ae9e897015e8a3e0447a91b8528ba186bdeb Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 18 Sep 2022 15:07:30 +0900
Subject: [PATCH 2807/3199] save

---
 runtime/src/bank.rs | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 6ea3b0d94f7b22..7a1b5f875303da 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1306,8 +1306,6 @@ impl Scheduler {
                 thread_priority::set_current_thread_priority(thread_priority::ThreadPriority::Max).unwrap();
             }
 
-            let current_thread_name = std::thread::current().name().unwrap().to_string();
-
             while let Ok(solana_scheduler::ExecutablePayload(mut ee)) = scheduled_ee_receiver.recv() {
                 let (mut wall_time, cpu_time) = (Measure::start("process_message_time"), cpu_time::ThreadTime::now());
 
@@ -1363,6 +1361,7 @@ impl Scheduler {
 
                 ee.finish_time = std::time::SystemTime::now();
                 ee.execution_result = Some(tx_result);
+                ee.thx = thx;
                 wall_time.stop();
                 ee.execution_us = wall_time.as_us();
                 ee.execution_cpu_us = cpu_time.elapsed().as_micros();
@@ -1394,7 +1393,7 @@ impl Scheduler {
                         "individual_tx_stats",
                         ("slot", slot, i64),
                         ("index", transaction_index, i64),
-                        ("thread", current_thread_name, String),
+                        ("thread", format!("solScExLane{:02}", ee.thx), String),
                         ("signature", &sig, String),
                         ("account_locks_in_json", "{}", String),
                         ("status", ee.execution_result.unwrap(), String),

From 58fa132b16f29c71d46871a8ff1373b287ffaf70 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 18 Sep 2022 15:08:37 +0900
Subject: [PATCH 2808/3199] save

---
 runtime/src/bank.rs  | 4 ++--
 scheduler/src/lib.rs | 4 ++++
 2 files changed, 6 insertions(+), 2 deletions(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 7a1b5f875303da..f24645e82a9ce3 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1391,8 +1391,8 @@ impl Scheduler {
                     datapoint_info_at!(
                         ee.finish_time,
                         "individual_tx_stats",
-                        ("slot", slot, i64),
-                        ("index", transaction_index, i64),
+                        ("slot", ee.slot, i64),
+                        ("index", ee.transaction_index, i64),
                         ("thread", format!("solScExLane{:02}", ee.thx), String),
                         ("signature", &sig, String),
                         ("account_locks_in_json", "{}", String),
diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 10c4caba1a407d..141905c2452916 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -37,6 +37,10 @@ pub struct ExecutionEnvironment {
     pub finalized_lock_attempts: Vec<LockAttempt>,
     pub is_reindexed: bool,
     pub execution_result: Option<Result<(), solana_sdk::transaction::TransactionError>>,
+    pub thx: usize,
+    pub slot: Slot,
+    pub execution_us: usize,
+    pub execution_cpu_us: usize,
 }
 
 impl ExecutionEnvironment {

From 2ffff50deddd86b1a74ceb5e922d9e6588f554ff Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 18 Sep 2022 15:09:31 +0900
Subject: [PATCH 2809/3199] save

---
 scheduler/src/lib.rs | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 141905c2452916..1fee534f6f4730 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1331,6 +1331,10 @@ impl ScheduleStage {
             finalized_lock_attempts,
             is_reindexed: Default::default(),
             execution_result: Default::default(),
+            thx: Default::default(),
+            transaction_index: Default::default(),
+            execution_us: Default::default(),
+            execution_cpu_us: Default::default(),
         })
     }
 

From 0662a1a1412350e91c061397bd86081dcebb9c02 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 18 Sep 2022 15:09:47 +0900
Subject: [PATCH 2810/3199] save

---
 scheduler/src/lib.rs | 1 +
 1 file changed, 1 insertion(+)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 1fee534f6f4730..e23e7af7d386c5 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -38,6 +38,7 @@ pub struct ExecutionEnvironment {
     pub is_reindexed: bool,
     pub execution_result: Option<Result<(), solana_sdk::transaction::TransactionError>>,
     pub thx: usize,
+    pub transaction_index: usize,
     pub slot: Slot,
     pub execution_us: usize,
     pub execution_cpu_us: usize,

From 5c60dd2ffa4c10b672fce9cc74dfb52ef2da34ad Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 18 Sep 2022 15:10:02 +0900
Subject: [PATCH 2811/3199] save

---
 scheduler/src/lib.rs | 1 +
 1 file changed, 1 insertion(+)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index e23e7af7d386c5..f3f321ce545e51 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1336,6 +1336,7 @@ impl ScheduleStage {
             transaction_index: Default::default(),
             execution_us: Default::default(),
             execution_cpu_us: Default::default(),
+            slot: Default::default(),
         })
     }
 

From a3939a5a5543f8df28ba6e7a30c225beccbd215d Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 18 Sep 2022 15:10:29 +0900
Subject: [PATCH 2812/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index f3f321ce545e51..1dd9de4d2aa10d 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1336,7 +1336,7 @@ impl ScheduleStage {
             transaction_index: Default::default(),
             execution_us: Default::default(),
             execution_cpu_us: Default::default(),
-            slot: Default::default(),
+            slot: 0,
         })
     }
 

From 6cb184ce5353098d65bba29c04011a8201f921cd Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 18 Sep 2022 15:10:49 +0900
Subject: [PATCH 2813/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 1dd9de4d2aa10d..ddf4fb485fa81f 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -39,7 +39,7 @@ pub struct ExecutionEnvironment {
     pub execution_result: Option<Result<(), solana_sdk::transaction::TransactionError>>,
     pub thx: usize,
     pub transaction_index: usize,
-    pub slot: Slot,
+    pub slot: solana_sdk::clock::Slot,
     pub execution_us: usize,
     pub execution_cpu_us: usize,
 }

From 84dd678a0878b8b93a85cd0123a92d5e4ca41ea4 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 18 Sep 2022 15:11:22 +0900
Subject: [PATCH 2814/3199] save

---
 scheduler/src/lib.rs | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index ddf4fb485fa81f..c7c3b38500b4c9 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -42,6 +42,7 @@ pub struct ExecutionEnvironment {
     pub slot: solana_sdk::clock::Slot,
     pub execution_us: usize,
     pub execution_cpu_us: usize,
+    pub finish_time: std::time::SystemTime,
 }
 
 impl ExecutionEnvironment {
@@ -1336,6 +1337,7 @@ impl ScheduleStage {
             transaction_index: Default::default(),
             execution_us: Default::default(),
             execution_cpu_us: Default::default(),
+            finish_time: Default::default(),
             slot: 0,
         })
     }

From c223e9efdb2109555d61b15a36fc8ea4893ca833 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 18 Sep 2022 15:11:51 +0900
Subject: [PATCH 2815/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index c7c3b38500b4c9..308e1efd5ab39a 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -42,7 +42,7 @@ pub struct ExecutionEnvironment {
     pub slot: solana_sdk::clock::Slot,
     pub execution_us: usize,
     pub execution_cpu_us: usize,
-    pub finish_time: std::time::SystemTime,
+    pub finish_time: Option<std::time::SystemTime>,
 }
 
 impl ExecutionEnvironment {

From d3577e1fa18f87bdcc674d3d774848a1123d521c Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 18 Sep 2022 15:12:58 +0900
Subject: [PATCH 2816/3199] save

---
 runtime/src/bank.rs  | 4 ++--
 scheduler/src/lib.rs | 4 ++--
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index f24645e82a9ce3..03c2a35d0337fa 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1359,7 +1359,7 @@ impl Scheduler {
                     error!("found odd tx error: slot: {}, signature: {}, {:?}", slot, sig, tx_result);
                 };
 
-                ee.finish_time = std::time::SystemTime::now();
+                ee.finish_time = Some(std::time::SystemTime::now());
                 ee.execution_result = Some(tx_result);
                 ee.thx = thx;
                 wall_time.stop();
@@ -1389,7 +1389,7 @@ impl Scheduler {
 
 
                     datapoint_info_at!(
-                        ee.finish_time,
+                        ee.finish_time.unwrap(),
                         "individual_tx_stats",
                         ("slot", ee.slot, i64),
                         ("index", ee.transaction_index, i64),
diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 308e1efd5ab39a..98288670034835 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -40,8 +40,8 @@ pub struct ExecutionEnvironment {
     pub thx: usize,
     pub transaction_index: usize,
     pub slot: solana_sdk::clock::Slot,
-    pub execution_us: usize,
-    pub execution_cpu_us: usize,
+    pub execution_us: u64,
+    pub execution_cpu_us: u128,
     pub finish_time: Option<std::time::SystemTime>,
 }
 

From 2f1bec0e54b22ef5b418c27314f2191208ac2918 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 18 Sep 2022 15:13:29 +0900
Subject: [PATCH 2817/3199] save

---
 runtime/src/bank.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 03c2a35d0337fa..07a774ac0569c0 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1396,7 +1396,7 @@ impl Scheduler {
                         ("thread", format!("solScExLane{:02}", ee.thx), String),
                         ("signature", &sig, String),
                         ("account_locks_in_json", "{}", String),
-                        ("status", ee.execution_result.unwrap(), String),
+                        ("status", format!("{:?}", ee.execution_result.unwrap()), String),
                         ("duration", ee.execution_us, i64),
                         ("cpu_duration", ee.execution_cpu_us, i64),
                         ("compute_units", ee.cu, i64),

From ae217689ae684eb80e1737cdeac7f2601e1dfe76 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 18 Sep 2022 15:13:51 +0900
Subject: [PATCH 2818/3199] save

---
 runtime/src/bank.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 07a774ac0569c0..b92c32b9e18645 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1396,7 +1396,7 @@ impl Scheduler {
                         ("thread", format!("solScExLane{:02}", ee.thx), String),
                         ("signature", &sig, String),
                         ("account_locks_in_json", "{}", String),
-                        ("status", format!("{:?}", ee.execution_result.unwrap()), String),
+                        ("status", format!("{:?}", ee.execution_result.as_ref().unwrap()), String),
                         ("duration", ee.execution_us, i64),
                         ("cpu_duration", ee.execution_cpu_us, i64),
                         ("compute_units", ee.cu, i64),

From 20b8d611af2a2c9e65a0632735000fa9862e01fa Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 18 Sep 2022 15:15:38 +0900
Subject: [PATCH 2819/3199] save

---
 runtime/src/bank.rs  | 3 ++-
 scheduler/src/lib.rs | 2 +-
 2 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index b92c32b9e18645..5f3401ed5cfcf9 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1359,10 +1359,11 @@ impl Scheduler {
                     error!("found odd tx error: slot: {}, signature: {}, {:?}", slot, sig, tx_result);
                 };
 
-                ee.finish_time = Some(std::time::SystemTime::now());
                 ee.execution_result = Some(tx_result);
+                ee.finish_time = Some(std::time::SystemTime::now());
                 ee.thx = thx;
                 wall_time.stop();
+                ee.slot = slot;
                 ee.execution_us = wall_time.as_us();
                 ee.execution_cpu_us = cpu_time.elapsed().as_micros();
 
diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 98288670034835..6f104ebce2e855 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -37,12 +37,12 @@ pub struct ExecutionEnvironment {
     pub finalized_lock_attempts: Vec<LockAttempt>,
     pub is_reindexed: bool,
     pub execution_result: Option<Result<(), solana_sdk::transaction::TransactionError>>,
+    pub finish_time: Option<std::time::SystemTime>,
     pub thx: usize,
     pub transaction_index: usize,
     pub slot: solana_sdk::clock::Slot,
     pub execution_us: u64,
     pub execution_cpu_us: u128,
-    pub finish_time: Option<std::time::SystemTime>,
 }
 
 impl ExecutionEnvironment {

From 58ee1ac20a6b7954785398cf38a39175b894dbc2 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 18 Sep 2022 15:21:34 +0900
Subject: [PATCH 2820/3199] save

---
 runtime/src/bank.rs | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 5f3401ed5cfcf9..2f8cc0b57db938 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1362,8 +1362,9 @@ impl Scheduler {
                 ee.execution_result = Some(tx_result);
                 ee.finish_time = Some(std::time::SystemTime::now());
                 ee.thx = thx;
-                wall_time.stop();
+                ee.transaction_index = transaction_index;
                 ee.slot = slot;
+                wall_time.stop();
                 ee.execution_us = wall_time.as_us();
                 ee.execution_cpu_us = cpu_time.elapsed().as_micros();
 

From 43b8830227cea3316aaf56cedb848ca290d84ac1 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 18 Sep 2022 15:22:08 +0900
Subject: [PATCH 2821/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 6f104ebce2e855..ec47af879706c9 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -39,7 +39,7 @@ pub struct ExecutionEnvironment {
     pub execution_result: Option<Result<(), solana_sdk::transaction::TransactionError>>,
     pub finish_time: Option<std::time::SystemTime>,
     pub thx: usize,
-    pub transaction_index: usize,
+    pub transaction_index: u64,
     pub slot: solana_sdk::clock::Slot,
     pub execution_us: u64,
     pub execution_cpu_us: u128,

From 6b068c3040448eed8236a49e53829152c9fc6cdf Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 18 Sep 2022 15:47:27 +0900
Subject: [PATCH 2822/3199] save

---
 scheduler/src/lib.rs | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index ec47af879706c9..980a751bd4e1a7 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -96,7 +96,9 @@ impl ExecutionEnvironment {
                             break;
                         }
                     }
-                    assert!(!should_remove || removed);
+                    if should_remove && !removed {
+                        contended_unique_weights.remove_task(&uq);
+                    }
                     found.then(|| Task::clone_in_queue(task))
                 })
                 .flatten()

From dd031993dd9d4bda3de73e9fd2799ac33417992c Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 18 Sep 2022 22:18:36 +0900
Subject: [PATCH 2823/3199] save

---
 ledger/src/blockstore_processor.rs | 13 -------------
 1 file changed, 13 deletions(-)

diff --git a/ledger/src/blockstore_processor.rs b/ledger/src/blockstore_processor.rs
index 13b3ddef2fd1e5..102fdaed8fcda0 100644
--- a/ledger/src/blockstore_processor.rs
+++ b/ledger/src/blockstore_processor.rs
@@ -346,19 +346,6 @@ fn execute_batches(
         return Ok(());
     }
 
-    let ((lock_results, sanitized_txs), transaction_indexes): ((Vec<_>, Vec<_>), Vec<_>) = batches
-        .iter()
-        .flat_map(|batch| {
-            batch
-                .batch
-                .lock_results()
-                .iter()
-                .cloned()
-                .zip(batch.batch.sanitized_transactions().to_vec())
-                .zip(batch.transaction_indexes.to_vec())
-        })
-        .unzip();
-
     let mut minimal_tx_cost = u64::MAX;
     let mut total_cost: u64 = 0;
     let mut tx_batches: Vec<TransactionBatchWithIndexes> = vec![];

From 5db8381b693775e367b04d6d4680081dc10b8450 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 18 Sep 2022 22:20:18 +0900
Subject: [PATCH 2824/3199] save

---
 ledger/src/blockstore_processor.rs | 13 -------------
 1 file changed, 13 deletions(-)

diff --git a/ledger/src/blockstore_processor.rs b/ledger/src/blockstore_processor.rs
index 102fdaed8fcda0..5d60f49d7f7fb9 100644
--- a/ledger/src/blockstore_processor.rs
+++ b/ledger/src/blockstore_processor.rs
@@ -281,11 +281,6 @@ fn execute_batches_internal(
     bank: &Arc<Bank>,
     batches: &[TransactionBatchWithIndexes],
     entry_callback: Option<&ProcessCallback>,
-    transaction_status_sender: Option<&TransactionStatusSender>,
-    replay_vote_sender: Option<&ReplayVoteSender>,
-    cost_capacity_meter: Arc<RwLock<BlockCostCapacityMeter>>,
-    tx_costs: &[u64],
-    log_messages_bytes_limit: Option<usize>,
 ) -> Result<ExecuteBatchesInternalMetrics> {
     assert!(!batches.is_empty());
 
@@ -346,19 +341,11 @@ fn execute_batches(
         return Ok(());
     }
 
-    let mut minimal_tx_cost = u64::MAX;
-    let mut total_cost: u64 = 0;
-    let mut tx_batches: Vec<TransactionBatchWithIndexes> = vec![];
     let mut tx_batch_costs: Vec<u64> = vec![];
     let execute_batches_internal_metrics = execute_batches_internal(
         bank,
         batches,
         entry_callback,
-        transaction_status_sender,
-        replay_vote_sender,
-        cost_capacity_meter,
-        &tx_batch_costs,
-        log_messages_bytes_limit,
     )?;
 
     confirmation_timing.process_execute_batches_internal_metrics(execute_batches_internal_metrics);

From 26edeb028fa00db5131b5ac0b0621678b99050bc Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 18 Sep 2022 22:23:42 +0900
Subject: [PATCH 2825/3199] save

---
 runtime/src/bank.rs | 13 +++++++++++++
 1 file changed, 13 insertions(+)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 2f8cc0b57db938..319bec04e395ff 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -4340,6 +4340,19 @@ impl Bank {
         TransactionBatch::new(lock_results, self, Cow::Borrowed(txs))
     }
 
+    pub fn prepare_sanitized_batch_noop<'a, 'b>(
+        &'a self,
+        txs: &'b [SanitizedTransaction],
+    ) -> TransactionBatch<'a, 'b> {
+        let tx_account_lock_limit = self.get_transaction_account_lock_limit();
+        let locks_results: Vec<Result<_>> = txs
+            .map(|tx| tx.get_account_locks(tx_account_lock_limit))
+            .collect();
+        let batch = TransactionBatch::new(lock_results, self, Cow::Borrowed(txs));
+        batch.set_needs_unlock(false);
+        batch
+    }
+
     /// Prepare a locked transaction batch from a list of sanitized transactions, and their cost
     /// limited packing status
     pub fn prepare_sanitized_batch_with_results<'a, 'b>(

From 8b2cc0ae6cfa90a2906a0daaaa25b643456b11b6 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 18 Sep 2022 22:23:53 +0900
Subject: [PATCH 2826/3199] save

---
 runtime/src/bank.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 319bec04e395ff..15ff98a31bec36 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -4345,7 +4345,7 @@ impl Bank {
         txs: &'b [SanitizedTransaction],
     ) -> TransactionBatch<'a, 'b> {
         let tx_account_lock_limit = self.get_transaction_account_lock_limit();
-        let locks_results: Vec<Result<_>> = txs
+        let lock_results: Vec<Result<_>> = txs
             .map(|tx| tx.get_account_locks(tx_account_lock_limit))
             .collect();
         let batch = TransactionBatch::new(lock_results, self, Cow::Borrowed(txs));

From b8777ab989bc53cbedf38fe89f49618c6090e535 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 18 Sep 2022 22:24:13 +0900
Subject: [PATCH 2827/3199] save

---
 runtime/src/bank.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 15ff98a31bec36..10cb18faf959b2 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -4345,7 +4345,7 @@ impl Bank {
         txs: &'b [SanitizedTransaction],
     ) -> TransactionBatch<'a, 'b> {
         let tx_account_lock_limit = self.get_transaction_account_lock_limit();
-        let lock_results: Vec<Result<_>> = txs
+        let lock_results: Vec<Result<_>> = txs.iter()
             .map(|tx| tx.get_account_locks(tx_account_lock_limit))
             .collect();
         let batch = TransactionBatch::new(lock_results, self, Cow::Borrowed(txs));

From ec41f28b7cf3cf0080d8b3f9ead4ec4258bcab22 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 18 Sep 2022 22:24:48 +0900
Subject: [PATCH 2828/3199] save

---
 runtime/src/bank.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 10cb18faf959b2..3c2105d52fa5f5 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -4346,7 +4346,7 @@ impl Bank {
     ) -> TransactionBatch<'a, 'b> {
         let tx_account_lock_limit = self.get_transaction_account_lock_limit();
         let lock_results: Vec<Result<_>> = txs.iter()
-            .map(|tx| tx.get_account_locks(tx_account_lock_limit))
+            .map(|tx| tx.get_account_locks(tx_account_lock_limit).map(|_| ()))
             .collect();
         let batch = TransactionBatch::new(lock_results, self, Cow::Borrowed(txs));
         batch.set_needs_unlock(false);

From 96e8ad077d0ce75b902bd8bbeb15d1cb3e0b3b60 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 18 Sep 2022 22:25:01 +0900
Subject: [PATCH 2829/3199] save

---
 runtime/src/bank.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 3c2105d52fa5f5..dfe2a9de646de4 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -4348,7 +4348,7 @@ impl Bank {
         let lock_results: Vec<Result<_>> = txs.iter()
             .map(|tx| tx.get_account_locks(tx_account_lock_limit).map(|_| ()))
             .collect();
-        let batch = TransactionBatch::new(lock_results, self, Cow::Borrowed(txs));
+        let mut batch = TransactionBatch::new(lock_results, self, Cow::Borrowed(txs));
         batch.set_needs_unlock(false);
         batch
     }

From f22a48349f794c8dcd09245c384e95e0b4706e82 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 18 Sep 2022 22:25:39 +0900
Subject: [PATCH 2830/3199] save

---
 ledger/src/blockstore_processor.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ledger/src/blockstore_processor.rs b/ledger/src/blockstore_processor.rs
index 5d60f49d7f7fb9..fdd4b3242802cd 100644
--- a/ledger/src/blockstore_processor.rs
+++ b/ledger/src/blockstore_processor.rs
@@ -476,7 +476,7 @@ fn process_entries_with_callback(
 
                 loop {
                     // try to lock the accounts
-                    let batch = bank.prepare_sanitized_batch(transactions);
+                    let batch = bank.prepare_sanitized_batch_noop(transactions);
                     let first_lock_err = first_err(batch.lock_results());
 
                     // if locking worked

From ffd37a585489ab43ab8750c821d7ec396fa101e1 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 18 Sep 2022 22:29:42 +0900
Subject: [PATCH 2831/3199] save

---
 ledger/src/blockstore_processor.rs | 68 ++++++++----------------------
 1 file changed, 17 insertions(+), 51 deletions(-)

diff --git a/ledger/src/blockstore_processor.rs b/ledger/src/blockstore_processor.rs
index fdd4b3242802cd..8307540c041a91 100644
--- a/ledger/src/blockstore_processor.rs
+++ b/ledger/src/blockstore_processor.rs
@@ -474,57 +474,23 @@ fn process_entries_with_callback(
                     (starting_index..starting_index.saturating_add(transactions.len())).collect()
                 };
 
-                loop {
-                    // try to lock the accounts
-                    let batch = bank.prepare_sanitized_batch_noop(transactions);
-                    let first_lock_err = first_err(batch.lock_results());
-
-                    // if locking worked
-                    if first_lock_err.is_ok() {
-                        batches.push(TransactionBatchWithIndexes {
-                            batch,
-                            transaction_indexes,
-                        });
-                        // entry is scheduled to be processed, transactions in it can be used to
-                        // update prioritization fee cache asynchronously.
-                        prioritization_fee_cache.update(bank.clone(), transactions.iter());
-                        // done with this entry
-                        break;
-                    }
-                    // else we failed to lock, 2 possible reasons
-                    if batches.is_empty() {
-                        // An entry has account lock conflicts with *itself*, which should not happen
-                        // if generated by a properly functioning leader
-                        datapoint_error!(
-                            "validator_process_entry_error",
-                            (
-                                "error",
-                                format!(
-                                    "Lock accounts error, entry conflicts with itself, txs: {:?}",
-                                    transactions
-                                ),
-                                String
-                            )
-                        );
-                        // bail
-                        first_lock_err?;
-                    } else {
-                        // else we have an entry that conflicts with a prior entry
-                        // execute the current queue and try to process this entry again
-                        execute_batches(
-                            bank,
-                            &batches,
-                            entry_callback,
-                            transaction_status_sender,
-                            replay_vote_sender,
-                            confirmation_timing,
-                            cost_capacity_meter.clone(),
-                            &cost_model,
-                            log_messages_bytes_limit,
-                        )?;
-                        batches.clear();
-                    }
-                }
+                let batch = bank.prepare_sanitized_batch_noop(transactions);
+                batches.push(TransactionBatchWithIndexes {
+                    batch,
+                    transaction_indexes,
+                });
+                execute_batches(
+                    bank,
+                    &batches,
+                    entry_callback,
+                    transaction_status_sender,
+                    replay_vote_sender,
+                    confirmation_timing,
+                    cost_capacity_meter.clone(),
+                    &cost_model,
+                    log_messages_bytes_limit,
+                )?;
+                batches.clear();
             }
         }
     }

From c9c7c64b12aa6e9cf7cff71e0704a843f888b6ad Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 18 Sep 2022 22:46:21 +0900
Subject: [PATCH 2832/3199] save

---
 core/src/replay_stage.rs | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/core/src/replay_stage.rs b/core/src/replay_stage.rs
index 9106f1843639ad..4455c9fae5f27c 100644
--- a/core/src/replay_stage.rs
+++ b/core/src/replay_stage.rs
@@ -442,6 +442,10 @@ impl ReplayStage {
         let t_replay = Builder::new()
             .name("solScReplayStg".to_string())
             .spawn(move || {
+                let max_thread_priority = std::env::var("MAX_THREAD_PRIORITY").is_ok();
+                if max_thread_priority {
+                    thread_priority::set_current_thread_priority(thread_priority::ThreadPriority::Max).unwrap();
+                }
                 let verify_recyclers = VerifyRecyclers::default();
                 let _exit = Finalizer::new(exit.clone());
                 let mut identity_keypair = cluster_info.keypair().clone();

From 31a5e749ded8855c8a10cb0282dfa3daeacc28ab Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 18 Sep 2022 22:46:59 +0900
Subject: [PATCH 2833/3199] save

---
 core/Cargo.toml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/core/Cargo.toml b/core/Cargo.toml
index 8217582236eeb4..d8bd19e2c791af 100644
--- a/core/Cargo.toml
+++ b/core/Cargo.toml
@@ -63,6 +63,7 @@ solana-vote-program = { path = "../programs/vote", version = "=1.12.0" }
 sys-info = "0.9.1"
 tempfile = "3.3.0"
 thiserror = "1.0"
+thread-priority = "0.9.2"
 tokio = { version = "~1.14.1", features = ["full"] }
 trees = "0.4.2"
 

From 253199219ec27893af3c73b6cf03fb3e519bee7a Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 18 Sep 2022 22:47:15 +0900
Subject: [PATCH 2834/3199] save

---
 Cargo.lock | 1 +
 1 file changed, 1 insertion(+)

diff --git a/Cargo.lock b/Cargo.lock
index 446959eab55907..06f45095c2e548 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -5122,6 +5122,7 @@ dependencies = [
  "tempfile",
  "test-case",
  "thiserror",
+ "thread-priority",
  "tokio",
  "trees",
 ]

From d0e8573e8ec8f90fcae3916d66169758dadebbb1 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 18 Sep 2022 22:53:29 +0900
Subject: [PATCH 2835/3199] save

---
 runtime/src/account_rent_state.rs |  3 ---
 runtime/src/accounts_db.rs        | 24 ------------------------
 runtime/src/bank.rs               | 14 +++-----------
 3 files changed, 3 insertions(+), 38 deletions(-)

diff --git a/runtime/src/account_rent_state.rs b/runtime/src/account_rent_state.rs
index 74cbc5b81af5f1..1036f18db0c9bb 100644
--- a/runtime/src/account_rent_state.rs
+++ b/runtime/src/account_rent_state.rs
@@ -72,13 +72,10 @@ impl RentState {
 pub(crate) fn submit_rent_state_metrics(pre_rent_state: &RentState, post_rent_state: &RentState) {
     match (pre_rent_state, post_rent_state) {
         (&RentState::Uninitialized, &RentState::RentPaying { .. }) => {
-            inc_new_counter_info!("rent_paying_err-new_account", 1);
         }
         (&RentState::RentPaying { .. }, &RentState::RentPaying { .. }) => {
-            inc_new_counter_info!("rent_paying_ok-legacy", 1);
         }
         (_, &RentState::RentPaying { .. }) => {
-            inc_new_counter_info!("rent_paying_err-other", 1);
         }
         _ => {}
     }
diff --git a/runtime/src/accounts_db.rs b/runtime/src/accounts_db.rs
index 0cd4b1392eabc2..84a17d31871a8c 100644
--- a/runtime/src/accounts_db.rs
+++ b/runtime/src/accounts_db.rs
@@ -3386,7 +3386,6 @@ impl AccountsDb {
                 !store_ids.contains(&store.append_vec_id())
             });
             if remaining_stores > 1 {
-                inc_new_counter_info!("accounts_db_shrink_extra_stores", 1);
                 info!(
                     "after shrink, slot has extra stores: {}, {}",
                     slot, remaining_stores
@@ -3629,19 +3628,6 @@ impl AccountsDb {
             }
         }
         measure.stop();
-        inc_new_counter_debug!(
-            "shrink_select_top_sparse_storage_entries-ms",
-            measure.as_ms() as usize
-        );
-        inc_new_counter_debug!(
-            "shrink_select_top_sparse_storage_entries-seeds",
-            candidates_count
-        );
-        inc_new_counter_debug!(
-            "shrink_total_preliminary_candidate_stores",
-            total_candidate_stores
-        );
-
         (shrink_slots, shrink_slots_next_batch)
     }
 
@@ -4137,17 +4123,11 @@ impl AccountsDb {
                     let mut measure = Measure::start("shrink_candidate_slots-ms");
                     self.do_shrink_slot_stores(slot, slot_shrink_candidates.values());
                     measure.stop();
-                    inc_new_counter_info!("shrink_candidate_slots-ms", measure.as_ms() as usize);
                     slot_shrink_candidates.len()
                 })
                 .sum()
         });
         measure_shrink_all_candidates.stop();
-        inc_new_counter_info!(
-            "shrink_all_candidate_slots-ms",
-            measure_shrink_all_candidates.as_ms() as usize
-        );
-        inc_new_counter_info!("shrink_all_candidate_slots-count", shrink_candidates_count);
         let mut pended_counts: usize = 0;
         if let Some(shrink_slots_next_batch) = shrink_slots_next_batch {
             let mut shrink_slots = self.shrink_candidate_slots.lock().unwrap();
@@ -4156,7 +4136,6 @@ impl AccountsDb {
                 shrink_slots.entry(slot).or_default().extend(stores);
             }
         }
-        inc_new_counter_info!("shrink_pended_stores-count", pended_counts);
 
         num_candidates
     }
@@ -7634,7 +7613,6 @@ impl AccountsDb {
             }
         }
         measure.stop();
-        inc_new_counter_info!("remove_dead_slots_metadata-ms", measure.as_ms() as usize);
     }
 
     /// lookup each pubkey in 'purged_slot_pubkeys' and unref it in the accounts index
@@ -7751,7 +7729,6 @@ impl AccountsDb {
             purged_account_slots,
         );
         measure.stop();
-        inc_new_counter_info!("clean_stored_dead_slots-ms", measure.as_ms() as usize);
         self.clean_accounts_stats
             .clean_stored_dead_slots_us
             .fetch_add(measure.as_us(), Ordering::Relaxed);
@@ -9181,7 +9158,6 @@ impl AccountsDb {
 
         let count = self.shrink_stale_slot_v1(&mut candidates);
         measure.stop();
-        inc_new_counter_info!("stale_slot_shrink-ms", measure.as_ms() as usize);
 
         count
     }
diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index dfe2a9de646de4..bfdadf2b58179b 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -83,7 +83,6 @@ use {
         ThreadPool, ThreadPoolBuilder,
     },
     solana_measure::{measure, measure::Measure},
-    solana_metrics::{inc_new_counter_debug, inc_new_counter_info},
     solana_program_runtime::{
         accounts_data_meter::MAX_ACCOUNTS_DATA_LEN,
         compute_budget::{self, ComputeBudget},
@@ -4246,7 +4245,6 @@ impl Bank {
             "register_tick() working on a bank that is already frozen or is undergoing freezing!"
         );
 
-        inc_new_counter_debug!("bank-register_tick-registered", 1);
         if self.is_block_boundary(self.tick_height.load(Relaxed) + 1) {
             self.register_recent_blockhash(hash);
         }
@@ -4907,7 +4905,6 @@ impl Bank {
     ) -> LoadAndExecuteTransactionsOutput {
         let sanitized_txs = batch.sanitized_transactions();
         debug!("processing transactions: {}", sanitized_txs.len());
-        inc_new_counter_info!("bank-process_transactions", sanitized_txs.len());
         let mut error_counters = TransactionErrorMetrics::default();
 
         let retryable_transaction_indexes: Vec<_> = batch
@@ -5392,12 +5389,6 @@ impl Bank {
         self.increment_transaction_count(tx_count);
         self.increment_signature_count(signature_count);
 
-        inc_new_counter_info!(
-            "bank-process_transactions-txs",
-            committed_transactions_count as usize
-        );
-        inc_new_counter_info!("bank-process_transactions-sigs", signature_count as usize);
-
         if committed_with_failure_result_count > 0 {
             self.transaction_error_count
                 .fetch_add(committed_with_failure_result_count, Relaxed);
@@ -5889,7 +5880,6 @@ impl Bank {
                     if let Some(rent_paying_pubkeys) = rent_paying_pubkeys {
                         if !rent_paying_pubkeys.contains(pubkey) {
                             // inc counter instead of assert while we verify this is correct
-                            inc_new_counter_info!("unexpected-rent-paying-pubkey", 1);
                             warn!(
                                 "Collecting rent from unexpected pubkey: {}, slot: {}, parent_slot: {:?}, partition_index: {}, partition_from_pubkey: {}",
                                 pubkey,
@@ -7292,11 +7282,13 @@ impl Bank {
         }
 
         info!(
-            "bank frozen: {} hash: {} accounts_delta: {} signature_count: {} last_blockhash: {} capitalization: {}",
+            "bank frozen: {} (parent: {}) hash: {} accounts_delta: {} sigs: {} txs: {}, last_blockhash: {} capitalization: {}",
             self.slot(),
+            self.parent_slot(),
             hash,
             accounts_delta_hash.hash,
             self.signature_count(),
+            self.transaction_count(),
             self.last_blockhash(),
             self.capitalization(),
         );

From c37d464f320f0c9f01ca31c3e369f05ad33db305 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 18 Sep 2022 22:54:41 +0900
Subject: [PATCH 2836/3199] save

---
 runtime/src/rent_collector.rs | 6 ------
 1 file changed, 6 deletions(-)

diff --git a/runtime/src/rent_collector.rs b/runtime/src/rent_collector.rs
index bbcf8379cd0b81..7510f9227f1899 100644
--- a/runtime/src/rent_collector.rs
+++ b/runtime/src/rent_collector.rs
@@ -104,12 +104,6 @@ impl RentCollector {
             if account_rent_epoch != 0
                 && (account_rent_epoch + 1 < self.epoch || account_rent_epoch > self.epoch + 1)
             {
-                // this should not occur in a running validator
-                if due == 0 {
-                    inc_new_counter_info!("rent-collector-rent-epoch-range-large-exempt", 1);
-                } else {
-                    inc_new_counter_info!("rent-collector-rent-epoch-range-large-paying", 1);
-                }
             }
 
             RentDue::Paying(due)

From 236b8202cda66f52df4ccb40c77868c822082326 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 18 Sep 2022 23:21:18 +0900
Subject: [PATCH 2837/3199] save

---
 runtime/src/accounts_db.rs | 16 +---------------
 1 file changed, 1 insertion(+), 15 deletions(-)

diff --git a/runtime/src/accounts_db.rs b/runtime/src/accounts_db.rs
index 84a17d31871a8c..4e5c9033fbd5bb 100644
--- a/runtime/src/accounts_db.rs
+++ b/runtime/src/accounts_db.rs
@@ -7404,21 +7404,7 @@ impl AccountsDb {
             });
             reclaims
         };
-        if len > threshold {
-            let chunk_size = std::cmp::max(1, len / quarter_thread_count()); // # pubkeys/thread
-            let batches = 1 + len / chunk_size;
-            (0..batches)
-                .into_par_iter()
-                .map(|batch| {
-                    let start = batch * chunk_size;
-                    let end = std::cmp::min(start + chunk_size, len);
-                    update(start, end)
-                })
-                .flatten()
-                .collect::<Vec<_>>()
-        } else {
-            update(0, len)
-        }
+        update(0, len)
     }
 
     fn should_not_shrink(aligned_bytes: u64, total_bytes: u64, num_stores: usize) -> bool {

From 5ea87843555fcb5a413eb1e6b5f4a8b147721d62 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 19 Sep 2022 14:02:38 +0900
Subject: [PATCH 2838/3199] save

---
 ledger/src/blockstore_processor.rs | 49 ++----------------------------
 1 file changed, 2 insertions(+), 47 deletions(-)

diff --git a/ledger/src/blockstore_processor.rs b/ledger/src/blockstore_processor.rs
index 8307540c041a91..29fd5e533194a5 100644
--- a/ledger/src/blockstore_processor.rs
+++ b/ledger/src/blockstore_processor.rs
@@ -424,10 +424,7 @@ fn process_entries_with_callback(
     prioritization_fee_cache: &PrioritizationFeeCache,
 ) -> Result<()> {
     // accumulator for entries that can be processed in parallel
-    let mut batches = vec![];
     let mut tick_hashes = vec![];
-    let mut rng = thread_rng();
-    let cost_model = CostModel::new();
 
     for ReplayEntry {
         entry,
@@ -439,20 +436,6 @@ fn process_entries_with_callback(
                 // If it's a tick, save it for later
                 tick_hashes.push(hash);
                 if bank.is_block_boundary(bank.tick_height() + tick_hashes.len() as u64) {
-                    // If it's a tick that will cause a new blockhash to be created,
-                    // execute the group and register the tick
-                    execute_batches(
-                        bank,
-                        &batches,
-                        entry_callback,
-                        transaction_status_sender,
-                        replay_vote_sender,
-                        confirmation_timing,
-                        cost_capacity_meter.clone(),
-                        &cost_model,
-                        log_messages_bytes_limit,
-                    )?;
-                    batches.clear();
                     for hash in &tick_hashes {
                         bank.register_tick(hash);
                     }
@@ -461,50 +444,22 @@ fn process_entries_with_callback(
             }
             EntryType::Transactions(transactions) => {
                 let starting_index = *starting_index;
-                let transaction_indexes = if randomize {
-                    panic!("randomize isn't supported");
-                    let mut transactions_and_indexes: Vec<(SanitizedTransaction, usize)> =
-                        transactions.drain(..).zip(starting_index..).collect();
-                    transactions_and_indexes.shuffle(&mut rng);
-                    let (txs, indexes): (Vec<_>, Vec<_>) =
-                        transactions_and_indexes.into_iter().unzip();
-                    *transactions = txs;
-                    indexes
-                } else {
-                    (starting_index..starting_index.saturating_add(transactions.len())).collect()
-                };
+                let transaction_indexes = (starting_index..starting_index.saturating_add(transactions.len())).collect();
 
-                let batch = bank.prepare_sanitized_batch_noop(transactions);
-                batches.push(TransactionBatchWithIndexes {
-                    batch,
-                    transaction_indexes,
-                });
                 execute_batches(
                     bank,
-                    &batches,
+                    &(transactions, transaction_indexes),
                     entry_callback,
                     transaction_status_sender,
                     replay_vote_sender,
                     confirmation_timing,
                     cost_capacity_meter.clone(),
-                    &cost_model,
                     log_messages_bytes_limit,
                 )?;
                 batches.clear();
             }
         }
     }
-    execute_batches(
-        bank,
-        &batches,
-        entry_callback,
-        transaction_status_sender,
-        replay_vote_sender,
-        confirmation_timing,
-        cost_capacity_meter,
-        &cost_model,
-        log_messages_bytes_limit,
-    )?;
     for hash in tick_hashes {
         bank.register_tick(hash);
     }

From cfa8ce458d98758eeb4448a0f758b3cc22619ebe Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 19 Sep 2022 14:03:18 +0900
Subject: [PATCH 2839/3199] save

---
 ledger/src/blockstore_processor.rs | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/ledger/src/blockstore_processor.rs b/ledger/src/blockstore_processor.rs
index 29fd5e533194a5..2b613c2ef750bd 100644
--- a/ledger/src/blockstore_processor.rs
+++ b/ledger/src/blockstore_processor.rs
@@ -448,7 +448,8 @@ fn process_entries_with_callback(
 
                 execute_batches(
                     bank,
-                    &(transactions, transaction_indexes),
+                    transactions,
+                    transaction_indexes,
                     entry_callback,
                     transaction_status_sender,
                     replay_vote_sender,

From 9d86f9c11837d4bc1031d30f1f4b0f7b7319e39e Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 19 Sep 2022 14:03:27 +0900
Subject: [PATCH 2840/3199] save

---
 ledger/src/blockstore_processor.rs | 1 -
 1 file changed, 1 deletion(-)

diff --git a/ledger/src/blockstore_processor.rs b/ledger/src/blockstore_processor.rs
index 2b613c2ef750bd..cea1bddd886ecf 100644
--- a/ledger/src/blockstore_processor.rs
+++ b/ledger/src/blockstore_processor.rs
@@ -457,7 +457,6 @@ fn process_entries_with_callback(
                     cost_capacity_meter.clone(),
                     log_messages_bytes_limit,
                 )?;
-                batches.clear();
             }
         }
     }

From ee657bc44d0f1d9c92cf600caf58a450e633001d Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 19 Sep 2022 14:04:48 +0900
Subject: [PATCH 2841/3199] save

---
 ledger/src/blockstore_processor.rs | 1 +
 1 file changed, 1 insertion(+)

diff --git a/ledger/src/blockstore_processor.rs b/ledger/src/blockstore_processor.rs
index cea1bddd886ecf..7ca0b75e06627f 100644
--- a/ledger/src/blockstore_processor.rs
+++ b/ledger/src/blockstore_processor.rs
@@ -329,6 +329,7 @@ fn rebatch_transactions<'a>(
 fn execute_batches(
     bank: &Arc<Bank>,
     batches: &[TransactionBatchWithIndexes],
+    transaction_indexes: impl Iter<Item = usize>,
     entry_callback: Option<&ProcessCallback>,
     transaction_status_sender: Option<&TransactionStatusSender>,
     replay_vote_sender: Option<&ReplayVoteSender>,

From 1e185b281ea81b6a95861e8592f11f24b4b42c3a Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 19 Sep 2022 14:05:00 +0900
Subject: [PATCH 2842/3199] save

---
 ledger/src/blockstore_processor.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ledger/src/blockstore_processor.rs b/ledger/src/blockstore_processor.rs
index 7ca0b75e06627f..d127e3871fb259 100644
--- a/ledger/src/blockstore_processor.rs
+++ b/ledger/src/blockstore_processor.rs
@@ -329,7 +329,7 @@ fn rebatch_transactions<'a>(
 fn execute_batches(
     bank: &Arc<Bank>,
     batches: &[TransactionBatchWithIndexes],
-    transaction_indexes: impl Iter<Item = usize>,
+    transaction_indexes: impl Iterarator<Item = usize>,
     entry_callback: Option<&ProcessCallback>,
     transaction_status_sender: Option<&TransactionStatusSender>,
     replay_vote_sender: Option<&ReplayVoteSender>,

From 8ccb728391a0496f0219e4d8da54de470eef85f5 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 19 Sep 2022 14:05:09 +0900
Subject: [PATCH 2843/3199] save

---
 ledger/src/blockstore_processor.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ledger/src/blockstore_processor.rs b/ledger/src/blockstore_processor.rs
index d127e3871fb259..f43b183325aadd 100644
--- a/ledger/src/blockstore_processor.rs
+++ b/ledger/src/blockstore_processor.rs
@@ -329,7 +329,7 @@ fn rebatch_transactions<'a>(
 fn execute_batches(
     bank: &Arc<Bank>,
     batches: &[TransactionBatchWithIndexes],
-    transaction_indexes: impl Iterarator<Item = usize>,
+    transaction_indexes: impl Iterator<Item = usize>,
     entry_callback: Option<&ProcessCallback>,
     transaction_status_sender: Option<&TransactionStatusSender>,
     replay_vote_sender: Option<&ReplayVoteSender>,

From c02274cbcf6cff681e1f7cdf1d28ddd9965afcb4 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 19 Sep 2022 14:05:21 +0900
Subject: [PATCH 2844/3199] save

---
 ledger/src/blockstore_processor.rs | 1 -
 1 file changed, 1 deletion(-)

diff --git a/ledger/src/blockstore_processor.rs b/ledger/src/blockstore_processor.rs
index f43b183325aadd..3a07e6de53c411 100644
--- a/ledger/src/blockstore_processor.rs
+++ b/ledger/src/blockstore_processor.rs
@@ -335,7 +335,6 @@ fn execute_batches(
     replay_vote_sender: Option<&ReplayVoteSender>,
     confirmation_timing: &mut ConfirmationTiming,
     cost_capacity_meter: Arc<RwLock<BlockCostCapacityMeter>>,
-    cost_model: &CostModel,
     log_messages_bytes_limit: Option<usize>,
 ) -> Result<()> {
     if batches.is_empty() {

From f0419c5aa1b6149a367e3df06d4b3c16c0e12d0e Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 19 Sep 2022 14:05:41 +0900
Subject: [PATCH 2845/3199] save

---
 ledger/src/blockstore_processor.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ledger/src/blockstore_processor.rs b/ledger/src/blockstore_processor.rs
index 3a07e6de53c411..0cf53222fc2a42 100644
--- a/ledger/src/blockstore_processor.rs
+++ b/ledger/src/blockstore_processor.rs
@@ -444,7 +444,7 @@ fn process_entries_with_callback(
             }
             EntryType::Transactions(transactions) => {
                 let starting_index = *starting_index;
-                let transaction_indexes = (starting_index..starting_index.saturating_add(transactions.len())).collect();
+                let transaction_indexes = (starting_index..starting_index.saturating_add(transactions.len())).iter();
 
                 execute_batches(
                     bank,

From f5a5f0b463c260743d3bc04a5c56399a358bf76e Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 19 Sep 2022 14:05:51 +0900
Subject: [PATCH 2846/3199] save

---
 ledger/src/blockstore_processor.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ledger/src/blockstore_processor.rs b/ledger/src/blockstore_processor.rs
index 0cf53222fc2a42..4105b0e724f110 100644
--- a/ledger/src/blockstore_processor.rs
+++ b/ledger/src/blockstore_processor.rs
@@ -444,7 +444,7 @@ fn process_entries_with_callback(
             }
             EntryType::Transactions(transactions) => {
                 let starting_index = *starting_index;
-                let transaction_indexes = (starting_index..starting_index.saturating_add(transactions.len())).iter();
+                let transaction_indexes = (starting_index..starting_index.saturating_add(transactions.len())).into_iter();
 
                 execute_batches(
                     bank,

From 2b77f51263fa09ac7367856fd1d6d75239774d75 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 19 Sep 2022 14:06:29 +0900
Subject: [PATCH 2847/3199] save

---
 ledger/src/blockstore_processor.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ledger/src/blockstore_processor.rs b/ledger/src/blockstore_processor.rs
index 4105b0e724f110..a43c2d7911f697 100644
--- a/ledger/src/blockstore_processor.rs
+++ b/ledger/src/blockstore_processor.rs
@@ -328,7 +328,7 @@ fn rebatch_transactions<'a>(
 #[inline(never)]
 fn execute_batches(
     bank: &Arc<Bank>,
-    batches: &[TransactionBatchWithIndexes],
+    batches: &[SanitizedTransaction],
     transaction_indexes: impl Iterator<Item = usize>,
     entry_callback: Option<&ProcessCallback>,
     transaction_status_sender: Option<&TransactionStatusSender>,

From 3c6c6ed5caf3b12078372d70143e5d835acfd474 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 19 Sep 2022 14:09:00 +0900
Subject: [PATCH 2848/3199] save

---
 ledger/src/blockstore_processor.rs | 45 +++++++++++++-----------------
 1 file changed, 19 insertions(+), 26 deletions(-)

diff --git a/ledger/src/blockstore_processor.rs b/ledger/src/blockstore_processor.rs
index a43c2d7911f697..256615ca2d1c9b 100644
--- a/ledger/src/blockstore_processor.rs
+++ b/ledger/src/blockstore_processor.rs
@@ -165,13 +165,10 @@ fn aggregate_total_execution_units(execute_timings: &ExecuteTimings) -> u64 {
 }
 
 fn execute_batch(
-    batch: &TransactionBatchWithIndexes,
+    transactions: &[SanitizedTransaction],
+    transaction_indexes: impl Iterator<Item = usize>,
     bank: &Arc<Bank>,
 ) -> Result<()> {
-    let TransactionBatchWithIndexes {
-        batch,
-        transaction_indexes,
-    } = batch;
     //let record_token_balances = transaction_status_sender.is_some();
 
     //let mut mint_decimals: HashMap<Pubkey, u8> = HashMap::new();
@@ -184,12 +181,12 @@ fn execute_batch(
     };
     */
 
-    batch.bank().schedule_and_commit_transactions(
-        batch,
+    bank.schedule_and_commit_transactions(
         bank,
+        transactions,
         transaction_indexes,
     );
-    if let Some(first_error_from_scheduler) = batch.bank().handle_aborted_transactions().into_iter().next() {
+    if let Some(first_error_from_scheduler) = bank.handle_aborted_transactions().into_iter().next() {
         first_error_from_scheduler?
     }
     return Ok(());
@@ -279,27 +276,27 @@ struct ExecuteBatchesInternalMetrics {
 
 fn execute_batches_internal(
     bank: &Arc<Bank>,
-    batches: &[TransactionBatchWithIndexes],
+    transactions: &[SanitizedTransaction],
+    transaction_indexes: impl Iterator<Item = usize>,
     entry_callback: Option<&ProcessCallback>,
 ) -> Result<ExecuteBatchesInternalMetrics> {
     assert!(!batches.is_empty());
 
     let mut execute_batches_elapsed = Measure::start("execute_batches_elapsed");
-    for batch in batches {
-        let result = execute_batch(
-            batch,
-            bank,
-        );
-        if let Some(entry_callback) = entry_callback {
-            entry_callback(bank);
-        }
-        result?
+    let result = execute_batch(
+        transactions,
+        transaction_indexes
+        bank,
+    );
+    if let Some(entry_callback) = entry_callback {
+        entry_callback(bank);
     }
+    result?
     execute_batches_elapsed.stop();
 
     Ok(ExecuteBatchesInternalMetrics {
         execution_timings_per_thread: Default::default(),
-        total_batches_len: batches.len() as u64,
+        total_batches_len: 1
         execute_batches_us: execute_batches_elapsed.as_us(),
     })
 }
@@ -328,7 +325,7 @@ fn rebatch_transactions<'a>(
 #[inline(never)]
 fn execute_batches(
     bank: &Arc<Bank>,
-    batches: &[SanitizedTransaction],
+    transactions: &[SanitizedTransaction],
     transaction_indexes: impl Iterator<Item = usize>,
     entry_callback: Option<&ProcessCallback>,
     transaction_status_sender: Option<&TransactionStatusSender>,
@@ -337,14 +334,10 @@ fn execute_batches(
     cost_capacity_meter: Arc<RwLock<BlockCostCapacityMeter>>,
     log_messages_bytes_limit: Option<usize>,
 ) -> Result<()> {
-    if batches.is_empty() {
-        return Ok(());
-    }
-
-    let mut tx_batch_costs: Vec<u64> = vec![];
     let execute_batches_internal_metrics = execute_batches_internal(
         bank,
-        batches,
+        transactions,
+        transaction_indexes,
         entry_callback,
     )?;
 

From 52ce455b67d0199ed7e27230c3448f2a82ca2433 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 19 Sep 2022 14:09:19 +0900
Subject: [PATCH 2849/3199] save

---
 ledger/src/blockstore_processor.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ledger/src/blockstore_processor.rs b/ledger/src/blockstore_processor.rs
index 256615ca2d1c9b..02b560b5844e1c 100644
--- a/ledger/src/blockstore_processor.rs
+++ b/ledger/src/blockstore_processor.rs
@@ -285,7 +285,7 @@ fn execute_batches_internal(
     let mut execute_batches_elapsed = Measure::start("execute_batches_elapsed");
     let result = execute_batch(
         transactions,
-        transaction_indexes
+        transaction_indexes,
         bank,
     );
     if let Some(entry_callback) = entry_callback {

From 72365652d97315800cd2b09481b5add3ccf97d6b Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 19 Sep 2022 14:09:29 +0900
Subject: [PATCH 2850/3199] save

---
 ledger/src/blockstore_processor.rs | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/ledger/src/blockstore_processor.rs b/ledger/src/blockstore_processor.rs
index 02b560b5844e1c..74471a33efc364 100644
--- a/ledger/src/blockstore_processor.rs
+++ b/ledger/src/blockstore_processor.rs
@@ -291,7 +291,8 @@ fn execute_batches_internal(
     if let Some(entry_callback) = entry_callback {
         entry_callback(bank);
     }
-    result?
+    result?;
+
     execute_batches_elapsed.stop();
 
     Ok(ExecuteBatchesInternalMetrics {

From 117539e705a253ad218c5a42f01491e32cccccb8 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 19 Sep 2022 14:09:37 +0900
Subject: [PATCH 2851/3199] save

---
 ledger/src/blockstore_processor.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ledger/src/blockstore_processor.rs b/ledger/src/blockstore_processor.rs
index 74471a33efc364..73eda57ce071a8 100644
--- a/ledger/src/blockstore_processor.rs
+++ b/ledger/src/blockstore_processor.rs
@@ -297,7 +297,7 @@ fn execute_batches_internal(
 
     Ok(ExecuteBatchesInternalMetrics {
         execution_timings_per_thread: Default::default(),
-        total_batches_len: 1
+        total_batches_len: 1,
         execute_batches_us: execute_batches_elapsed.as_us(),
     })
 }

From b940d20c7eb118e63b56969354eeb3ce6d180bd6 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 19 Sep 2022 14:09:48 +0900
Subject: [PATCH 2852/3199] save

---
 ledger/src/blockstore_processor.rs | 2 --
 1 file changed, 2 deletions(-)

diff --git a/ledger/src/blockstore_processor.rs b/ledger/src/blockstore_processor.rs
index 73eda57ce071a8..a94ebbd7453def 100644
--- a/ledger/src/blockstore_processor.rs
+++ b/ledger/src/blockstore_processor.rs
@@ -280,8 +280,6 @@ fn execute_batches_internal(
     transaction_indexes: impl Iterator<Item = usize>,
     entry_callback: Option<&ProcessCallback>,
 ) -> Result<ExecuteBatchesInternalMetrics> {
-    assert!(!batches.is_empty());
-
     let mut execute_batches_elapsed = Measure::start("execute_batches_elapsed");
     let result = execute_batch(
         transactions,

From a76961d81a9a6fd13d64589c30681b614aae66b5 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 19 Sep 2022 14:11:27 +0900
Subject: [PATCH 2853/3199] save

---
 runtime/src/bank.rs | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index bfdadf2b58179b..9c216be6a61329 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -6580,11 +6580,11 @@ impl Bank {
 
     pub fn schedule_and_commit_transactions(
         &self,
-        batch: &TransactionBatch,
-        bank: &Arc<Bank>,
-        transaction_indexes: &Vec<usize>,
+        this_arced_bank: &Arc<Bank>,
+        transactions: &[SanitizedTransaction],
+        transaction_indexes: impl Iterator<Item = usize>,
     ) {
-        assert_eq!(bank.slot(), batch.bank().slot());
+        assert_eq!(this_arced_bank.slot(), batch.bank().slot());
         trace!("schedule_and_commit_transactions(): [{:?}; {}]", transaction_indexes.iter().next(), transaction_indexes.len());
 
         let s = {
@@ -6597,7 +6597,7 @@ impl Bank {
                 // safe.
                 let ss = self.scheduler2.write().unwrap();
                 let w = ss.as_ref().unwrap();
-                *w.bank.write().unwrap() = Some(Arc::downgrade(&bank));
+                *w.bank.write().unwrap() = Some(Arc::downgrade(&this_arced_bank));
                 w.slot.store(bank.slot(), std::sync::atomic::Ordering::SeqCst);
                 drop(w);
                 drop(ss);
@@ -6606,13 +6606,13 @@ impl Bank {
                 info!("reconfigured scheduler to the bank slot: {}", self.slot());
                 s
             } else {
-                assert_eq!(bank.slot(), r.as_ref().unwrap().slot.load(std::sync::atomic::Ordering::SeqCst));
+                assert_eq!(this_arced_bank.slot(), r.as_ref().unwrap().slot.load(std::sync::atomic::Ordering::SeqCst));
                 r
             }
         };
         let scheduler = s.as_ref().unwrap();
 
-        for (st, &i) in batch.sanitized_transactions().iter().zip(transaction_indexes.iter()) {
+        for (st, &i) in transactions.zip(transaction_indexes) {
             scheduler.schedule(st, i);
         }
     }

From 648901a019c1d44a7b80f11dce84b1158d5a6281 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 19 Sep 2022 14:11:40 +0900
Subject: [PATCH 2854/3199] save

---
 runtime/src/bank.rs | 1 -
 1 file changed, 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 9c216be6a61329..c29e2b599fbe43 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -6584,7 +6584,6 @@ impl Bank {
         transactions: &[SanitizedTransaction],
         transaction_indexes: impl Iterator<Item = usize>,
     ) {
-        assert_eq!(this_arced_bank.slot(), batch.bank().slot());
         trace!("schedule_and_commit_transactions(): [{:?}; {}]", transaction_indexes.iter().next(), transaction_indexes.len());
 
         let s = {

From 304b7f176fa7bd205a197bf375a7916f25a4acc2 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 19 Sep 2022 14:12:27 +0900
Subject: [PATCH 2855/3199] save

---
 runtime/src/bank.rs | 1 +
 1 file changed, 1 insertion(+)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index c29e2b599fbe43..9575bc3608978f 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -6584,6 +6584,7 @@ impl Bank {
         transactions: &[SanitizedTransaction],
         transaction_indexes: impl Iterator<Item = usize>,
     ) {
+        assert_eq!(this_arced_bank.slot(), self.slot());
         trace!("schedule_and_commit_transactions(): [{:?}; {}]", transaction_indexes.iter().next(), transaction_indexes.len());
 
         let s = {

From 0e0712941f6021738deb5f9deb0e6a7e3e704871 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 19 Sep 2022 14:12:47 +0900
Subject: [PATCH 2856/3199] save

---
 runtime/src/bank.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 9575bc3608978f..2aacfbe488b834 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -6612,7 +6612,7 @@ impl Bank {
         };
         let scheduler = s.as_ref().unwrap();
 
-        for (st, &i) in transactions.zip(transaction_indexes) {
+        for (st, &i) in transactions.iter().zip(transaction_indexes) {
             scheduler.schedule(st, i);
         }
     }

From bdefa3e1fc98f017a1f54b2c3c1f1668ae31837c Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 19 Sep 2022 14:13:04 +0900
Subject: [PATCH 2857/3199] save

---
 runtime/src/bank.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 2aacfbe488b834..b230ad7d21db53 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -6598,7 +6598,7 @@ impl Bank {
                 let ss = self.scheduler2.write().unwrap();
                 let w = ss.as_ref().unwrap();
                 *w.bank.write().unwrap() = Some(Arc::downgrade(&this_arced_bank));
-                w.slot.store(bank.slot(), std::sync::atomic::Ordering::SeqCst);
+                w.slot.store(this_arced_bank.slot(), std::sync::atomic::Ordering::SeqCst);
                 drop(w);
                 drop(ss);
 

From 143de2e4f95625da57bc48e7ea193591f70a8c31 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 19 Sep 2022 14:13:20 +0900
Subject: [PATCH 2858/3199] save

---
 runtime/src/bank.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index b230ad7d21db53..bbcff9a10513e6 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -6612,7 +6612,7 @@ impl Bank {
         };
         let scheduler = s.as_ref().unwrap();
 
-        for (st, &i) in transactions.iter().zip(transaction_indexes) {
+        for (st, i) in transactions.iter().zip(transaction_indexes) {
             scheduler.schedule(st, i);
         }
     }

From 199e4cc4b99597de15978fc59760360e43b4a3f4 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 19 Sep 2022 14:14:50 +0900
Subject: [PATCH 2859/3199] save

---
 runtime/src/bank.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index bbcff9a10513e6..fcb26a6d7f0ad7 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -6585,7 +6585,7 @@ impl Bank {
         transaction_indexes: impl Iterator<Item = usize>,
     ) {
         assert_eq!(this_arced_bank.slot(), self.slot());
-        trace!("schedule_and_commit_transactions(): [{:?}; {}]", transaction_indexes.iter().next(), transaction_indexes.len());
+        trace!("schedule_and_commit_transactions(): [{:?}; {}]", transactions.len());
 
         let s = {
             let r = self.scheduler2.read().unwrap();

From 74742a487b2252cf1561ccdb140858099560d741 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 19 Sep 2022 14:15:07 +0900
Subject: [PATCH 2860/3199] save

---
 runtime/src/bank.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index fcb26a6d7f0ad7..14e0f7bdb22989 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -6585,7 +6585,7 @@ impl Bank {
         transaction_indexes: impl Iterator<Item = usize>,
     ) {
         assert_eq!(this_arced_bank.slot(), self.slot());
-        trace!("schedule_and_commit_transactions(): [{:?}; {}]", transactions.len());
+        trace!("schedule_and_commit_transactions(): {} txs", transactions.len());
 
         let s = {
             let r = self.scheduler2.read().unwrap();

From eabeb1b09ef9c469dbb056eca8f56d9952c57348 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 19 Sep 2022 15:00:03 +0900
Subject: [PATCH 2861/3199] save

---
 runtime/src/accounts_db.rs | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/runtime/src/accounts_db.rs b/runtime/src/accounts_db.rs
index 4e5c9033fbd5bb..e48bad721ec352 100644
--- a/runtime/src/accounts_db.rs
+++ b/runtime/src/accounts_db.rs
@@ -2083,7 +2083,7 @@ impl AccountsDb {
             new.temp_paths = Some(temp_dirs);
         };
 
-        new.start_background_hasher();
+        //new.start_background_hasher();
         {
             for path in new.paths.iter() {
                 std::fs::create_dir_all(path).expect("Create directory failed.");
@@ -2335,6 +2335,7 @@ impl AccountsDb {
         }
     }
 
+    /*
     fn start_background_hasher(&mut self) {
         let (sender, receiver) = unbounded();
         Builder::new()
@@ -2345,6 +2346,7 @@ impl AccountsDb {
             .unwrap();
         self.sender_bg_hasher = Some(sender);
     }
+    */
 
     pub(crate) fn purge_keys_exact<'a, C: 'a>(
         &'a self,

From 2ecb1e7c4e8f3127337ddc2598f053f9f4b5ed6e Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 19 Sep 2022 21:28:02 +0900
Subject: [PATCH 2862/3199] save

---
 runtime/src/bank.rs | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 14e0f7bdb22989..a4a72122806319 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1363,9 +1363,10 @@ impl Scheduler {
                 ee.thx = thx;
                 ee.transaction_index = transaction_index;
                 ee.slot = slot;
+                ee.execution_cpu_us = cpu_time.elapsed().as_micros();
+                // make wall time is longer than cpu time, always
                 wall_time.stop();
                 ee.execution_us = wall_time.as_us();
-                ee.execution_cpu_us = cpu_time.elapsed().as_micros();
 
                 //ee.reindex_with_address_book();
                 processed_ee_sender.send(solana_scheduler::UnlockablePayload(ee)).unwrap();

From 5a4713438f2d31189dbc20e09d0affa28bc17754 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 19 Sep 2022 21:34:33 +0900
Subject: [PATCH 2863/3199] save

---
 runtime/src/bank.rs | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index a4a72122806319..a0b19cd7715f66 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1281,6 +1281,7 @@ impl Scheduler {
         let preloader = Arc::new(address_book.preloader());
         let (transaction_sender, transaction_receiver) = crossbeam_channel::unbounded();
         let (scheduled_ee_sender, scheduled_ee_receiver) = crossbeam_channel::unbounded();
+        let (scheduled_high_ee_sender, scheduled_high_ee_receiver) = crossbeam_channel::unbounded();
         let (processed_ee_sender, processed_ee_receiver) = crossbeam_channel::unbounded();
         let (retired_ee_sender, retired_ee_receiver) = crossbeam_channel::unbounded();
 
@@ -1296,7 +1297,7 @@ impl Scheduler {
         let max_thread_priority = std::env::var("MAX_THREAD_PRIORITY").is_ok();
 
         let executing_thread_handles = (0..executing_thread_count).map(|thx| {
-            let (scheduled_ee_receiver, processed_ee_sender) = (scheduled_ee_receiver.clone(), processed_ee_sender.clone());
+            let (scheduled_ee_receiver, scheduled_high_ee_receiver, processed_ee_sender) = (scheduled_ee_receiver.clone(), scheduled_high_ee_receiver.clone(), processed_ee_sender.clone());
             let bank = bank.clone();
 
             std::thread::Builder::new().name(format!("solScExLane{:02}", thx)).spawn(move || {
@@ -1305,7 +1306,8 @@ impl Scheduler {
                 thread_priority::set_current_thread_priority(thread_priority::ThreadPriority::Max).unwrap();
             }
 
-            while let Ok(solana_scheduler::ExecutablePayload(mut ee)) = scheduled_ee_receiver.recv() {
+
+            while let Ok(solana_scheduler::ExecutablePayload(mut ee)) = scheduled_high_ee_receiver.try_recv().map_err(|_e| scheduled_ee_receiver.recv()) {
                 let (mut wall_time, cpu_time) = (Measure::start("process_message_time"), cpu_time::ThreadTime::now());
 
                 let current_execute_clock = ee.task.execute_time();

From 40e1696638fa75aa144d480fc2d93abe287d23f8 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 19 Sep 2022 21:38:22 +0900
Subject: [PATCH 2864/3199] save

---
 runtime/src/bank.rs  | 2 ++
 scheduler/src/lib.rs | 3 +++
 2 files changed, 5 insertions(+)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index a0b19cd7715f66..5018f4ecd5b5cf 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1443,6 +1443,7 @@ impl Scheduler {
                     &mut address_book,
                     &transaction_receiver,
                     &scheduled_ee_sender,
+                    &scheduled_high_ee_sender,
                     &processed_ee_receiver,
                     Some(&retired_ee_sender),
                 );
@@ -1457,6 +1458,7 @@ impl Scheduler {
 
             drop(transaction_receiver);
             drop(scheduled_ee_sender);
+            drop(scheduled_high_ee_sender);
             drop(processed_ee_receiver);
 
             todo!();
diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 980a751bd4e1a7..a8d852bac5ddcd 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1438,6 +1438,7 @@ impl ScheduleStage {
         address_book: &mut AddressBook,
         mut from_prev: &'a crossbeam_channel::Receiver<SchedulablePayload>,
         to_execute_substage: &crossbeam_channel::Sender<ExecutablePayload>,
+        to_high_execute_substage: &crossbeam_channel::Sender<ExecutablePayload>,
         from_exec: &crossbeam_channel::Receiver<UnlockablePayload>,
         maybe_to_next_stage: Option<&crossbeam_channel::Sender<ExaminablePayload>>, // assume nonblocking
         never: &'a crossbeam_channel::Receiver<SchedulablePayload>,
@@ -1747,6 +1748,7 @@ impl ScheduleStage {
         address_book: &mut AddressBook,
         from: &crossbeam_channel::Receiver<SchedulablePayload>,
         to_execute_substage: &crossbeam_channel::Sender<ExecutablePayload>,
+        to_high_execute_substage: &crossbeam_channel::Sender<ExecutablePayload>,
         from_execute_substage: &crossbeam_channel::Receiver<UnlockablePayload>,
         maybe_to_next_stage: Option<&crossbeam_channel::Sender<ExaminablePayload>>, // assume nonblocking
     ) -> Option<std::sync::Arc<Checkpoint>> {
@@ -1762,6 +1764,7 @@ impl ScheduleStage {
             address_book,
             from,
             to_execute_substage,
+            to_high_execute_substage,
             from_execute_substage,
             maybe_to_next_stage,
             &crossbeam_channel::never(),

From 0b39f361a06e7c3d80e1719a298c72c40c404526 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 19 Sep 2022 21:39:02 +0900
Subject: [PATCH 2865/3199] save

---
 scheduler/src/lib.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index a8d852bac5ddcd..7fb8f0eee8577e 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1438,7 +1438,7 @@ impl ScheduleStage {
         address_book: &mut AddressBook,
         mut from_prev: &'a crossbeam_channel::Receiver<SchedulablePayload>,
         to_execute_substage: &crossbeam_channel::Sender<ExecutablePayload>,
-        to_high_execute_substage: &crossbeam_channel::Sender<ExecutablePayload>,
+        to_high_execute_substage: Option<&crossbeam_channel::Sender<ExecutablePayload>>,
         from_exec: &crossbeam_channel::Receiver<UnlockablePayload>,
         maybe_to_next_stage: Option<&crossbeam_channel::Sender<ExaminablePayload>>, // assume nonblocking
         never: &'a crossbeam_channel::Receiver<SchedulablePayload>,
@@ -1748,7 +1748,7 @@ impl ScheduleStage {
         address_book: &mut AddressBook,
         from: &crossbeam_channel::Receiver<SchedulablePayload>,
         to_execute_substage: &crossbeam_channel::Sender<ExecutablePayload>,
-        to_high_execute_substage: &crossbeam_channel::Sender<ExecutablePayload>,
+        to_high_execute_substage: Option<&crossbeam_channel::Sender<ExecutablePayload>>,
         from_execute_substage: &crossbeam_channel::Receiver<UnlockablePayload>,
         maybe_to_next_stage: Option<&crossbeam_channel::Sender<ExaminablePayload>>, // assume nonblocking
     ) -> Option<std::sync::Arc<Checkpoint>> {

From d4e03c3fde70114692f80d1a05cfdd45e191d53d Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 19 Sep 2022 21:39:16 +0900
Subject: [PATCH 2866/3199] save

---
 runtime/src/bank.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 5018f4ecd5b5cf..5b4c6832500368 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1443,7 +1443,7 @@ impl Scheduler {
                     &mut address_book,
                     &transaction_receiver,
                     &scheduled_ee_sender,
-                    &scheduled_high_ee_sender,
+                    Some(&scheduled_high_ee_sender),
                     &processed_ee_receiver,
                     Some(&retired_ee_sender),
                 );

From 2505d0c15144237cd5406343e83d8edeafe4a0ad Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 19 Sep 2022 21:39:50 +0900
Subject: [PATCH 2867/3199] save

---
 ledger-tool/src/main.rs | 1 +
 1 file changed, 1 insertion(+)

diff --git a/ledger-tool/src/main.rs b/ledger-tool/src/main.rs
index 2915f67e9e77d8..dc4f91fd2f5701 100644
--- a/ledger-tool/src/main.rs
+++ b/ledger-tool/src/main.rs
@@ -291,6 +291,7 @@ fn output_slot(
                 &mut address_book,
                 &muxed_receiver,
                 &pre_execute_env_sender,
+                None,
                 &post_execute_env_receiver,
                 Some(&post_schedule_env_sender),
             );

From e2254863e3fb3209520bd623af7f5a9d772093ab Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 19 Sep 2022 21:40:15 +0900
Subject: [PATCH 2868/3199] save

---
 ledger-tool/src/main.rs | 2 +-
 runtime/src/bank.rs     | 2 +-
 scheduler/src/lib.rs    | 4 ++--
 3 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/ledger-tool/src/main.rs b/ledger-tool/src/main.rs
index dc4f91fd2f5701..dd3f321305b33a 100644
--- a/ledger-tool/src/main.rs
+++ b/ledger-tool/src/main.rs
@@ -291,7 +291,7 @@ fn output_slot(
                 &mut address_book,
                 &muxed_receiver,
                 &pre_execute_env_sender,
-                None,
+                &pre_execute_env_sender,
                 &post_execute_env_receiver,
                 Some(&post_schedule_env_sender),
             );
diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 5b4c6832500368..5018f4ecd5b5cf 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1443,7 +1443,7 @@ impl Scheduler {
                     &mut address_book,
                     &transaction_receiver,
                     &scheduled_ee_sender,
-                    Some(&scheduled_high_ee_sender),
+                    &scheduled_high_ee_sender,
                     &processed_ee_receiver,
                     Some(&retired_ee_sender),
                 );
diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 7fb8f0eee8577e..a8d852bac5ddcd 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1438,7 +1438,7 @@ impl ScheduleStage {
         address_book: &mut AddressBook,
         mut from_prev: &'a crossbeam_channel::Receiver<SchedulablePayload>,
         to_execute_substage: &crossbeam_channel::Sender<ExecutablePayload>,
-        to_high_execute_substage: Option<&crossbeam_channel::Sender<ExecutablePayload>>,
+        to_high_execute_substage: &crossbeam_channel::Sender<ExecutablePayload>,
         from_exec: &crossbeam_channel::Receiver<UnlockablePayload>,
         maybe_to_next_stage: Option<&crossbeam_channel::Sender<ExaminablePayload>>, // assume nonblocking
         never: &'a crossbeam_channel::Receiver<SchedulablePayload>,
@@ -1748,7 +1748,7 @@ impl ScheduleStage {
         address_book: &mut AddressBook,
         from: &crossbeam_channel::Receiver<SchedulablePayload>,
         to_execute_substage: &crossbeam_channel::Sender<ExecutablePayload>,
-        to_high_execute_substage: Option<&crossbeam_channel::Sender<ExecutablePayload>>,
+        to_high_execute_substage: &crossbeam_channel::Sender<ExecutablePayload>,
         from_execute_substage: &crossbeam_channel::Receiver<UnlockablePayload>,
         maybe_to_next_stage: Option<&crossbeam_channel::Sender<ExaminablePayload>>, // assume nonblocking
     ) -> Option<std::sync::Arc<Checkpoint>> {

From f3759b1826a61d98e9d5922c2c978abbb366e779 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 19 Sep 2022 21:42:03 +0900
Subject: [PATCH 2869/3199] save

---
 ledger-tool/src/main.rs | 2 +-
 runtime/src/bank.rs     | 2 +-
 scheduler/src/lib.rs    | 4 ++--
 3 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/ledger-tool/src/main.rs b/ledger-tool/src/main.rs
index dd3f321305b33a..dc4f91fd2f5701 100644
--- a/ledger-tool/src/main.rs
+++ b/ledger-tool/src/main.rs
@@ -291,7 +291,7 @@ fn output_slot(
                 &mut address_book,
                 &muxed_receiver,
                 &pre_execute_env_sender,
-                &pre_execute_env_sender,
+                None,
                 &post_execute_env_receiver,
                 Some(&post_schedule_env_sender),
             );
diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 5018f4ecd5b5cf..5b4c6832500368 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1443,7 +1443,7 @@ impl Scheduler {
                     &mut address_book,
                     &transaction_receiver,
                     &scheduled_ee_sender,
-                    &scheduled_high_ee_sender,
+                    Some(&scheduled_high_ee_sender),
                     &processed_ee_receiver,
                     Some(&retired_ee_sender),
                 );
diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index a8d852bac5ddcd..7fb8f0eee8577e 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1438,7 +1438,7 @@ impl ScheduleStage {
         address_book: &mut AddressBook,
         mut from_prev: &'a crossbeam_channel::Receiver<SchedulablePayload>,
         to_execute_substage: &crossbeam_channel::Sender<ExecutablePayload>,
-        to_high_execute_substage: &crossbeam_channel::Sender<ExecutablePayload>,
+        to_high_execute_substage: Option<&crossbeam_channel::Sender<ExecutablePayload>>,
         from_exec: &crossbeam_channel::Receiver<UnlockablePayload>,
         maybe_to_next_stage: Option<&crossbeam_channel::Sender<ExaminablePayload>>, // assume nonblocking
         never: &'a crossbeam_channel::Receiver<SchedulablePayload>,
@@ -1748,7 +1748,7 @@ impl ScheduleStage {
         address_book: &mut AddressBook,
         from: &crossbeam_channel::Receiver<SchedulablePayload>,
         to_execute_substage: &crossbeam_channel::Sender<ExecutablePayload>,
-        to_high_execute_substage: &crossbeam_channel::Sender<ExecutablePayload>,
+        to_high_execute_substage: Option<&crossbeam_channel::Sender<ExecutablePayload>>,
         from_execute_substage: &crossbeam_channel::Receiver<UnlockablePayload>,
         maybe_to_next_stage: Option<&crossbeam_channel::Sender<ExaminablePayload>>, // assume nonblocking
     ) -> Option<std::sync::Arc<Checkpoint>> {

From f65fb99bbde653ce60a6e2fad898ddc7d8bfee67 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 19 Sep 2022 21:44:43 +0900
Subject: [PATCH 2870/3199] save

---
 scheduler/src/lib.rs | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 7fb8f0eee8577e..be5323718f223f 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1555,6 +1555,9 @@ impl ScheduleStage {
                                trace!("flushing2..: {:?} {} {} {} {}", (from_disconnected, from_exec_disconnected), runnable_queue.task_count(), contended_count, executing_queue_count, provisioning_tracker_count);
                                assert!(maybe_checkpoint.is_none());
                                maybe_checkpoint = Some(checkpoint);
+                                let delta = (processed_count - last_processed_count) as u128;
+                                let elapsed2 = elapsed.as_micros();
+                                info!("schedule_once:from_disconnected id_{:016x} ch(prev: {}, exec: {}|{}), r: {}, u/c: {}/{}, (imm+provi)/max: ({}+{})/{} s: {} l(s+f): {}+{} ({}txs/{}us={}tps)", random_id, (if from_disconnected { "-".to_string() } else { format!("{}", from_prev.len()) }), to_execute_substage.len(), from_exec.len(), runnable_queue.task_count(), address_book.uncontended_task_ids.len(), contended_count, executing_queue_count, provisioning_tracker_count, max_executing_queue_count, address_book.stuck_tasks.len(), processed_count, failed_lock_count, delta, elapsed.as_micros(), 1_000_000_u128*delta/elapsed2);
                            },
                            Err(_) => {
                                assert_eq!(from_prev.len(), 0);

From 17fcfd5d8cd2dbb1c4c6dff4ea4e6695336dfafe Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 19 Sep 2022 21:48:37 +0900
Subject: [PATCH 2871/3199] save

---
 scheduler/src/lib.rs | 7 ++-----
 1 file changed, 2 insertions(+), 5 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index be5323718f223f..481ee2fdcd8e1a 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1555,9 +1555,6 @@ impl ScheduleStage {
                                trace!("flushing2..: {:?} {} {} {} {}", (from_disconnected, from_exec_disconnected), runnable_queue.task_count(), contended_count, executing_queue_count, provisioning_tracker_count);
                                assert!(maybe_checkpoint.is_none());
                                maybe_checkpoint = Some(checkpoint);
-                                let delta = (processed_count - last_processed_count) as u128;
-                                let elapsed2 = elapsed.as_micros();
-                                info!("schedule_once:from_disconnected id_{:016x} ch(prev: {}, exec: {}|{}), r: {}, u/c: {}/{}, (imm+provi)/max: ({}+{})/{} s: {} l(s+f): {}+{} ({}txs/{}us={}tps)", random_id, (if from_disconnected { "-".to_string() } else { format!("{}", from_prev.len()) }), to_execute_substage.len(), from_exec.len(), runnable_queue.task_count(), address_book.uncontended_task_ids.len(), contended_count, executing_queue_count, provisioning_tracker_count, max_executing_queue_count, address_book.stuck_tasks.len(), processed_count, failed_lock_count, delta, elapsed.as_micros(), 1_000_000_u128*delta/elapsed2);
                            },
                            Err(_) => {
                                assert_eq!(from_prev.len(), 0);
@@ -1607,7 +1604,7 @@ impl ScheduleStage {
                     );
                     if let Some(ee) = maybe_ee {
                         executing_queue_count = executing_queue_count.checked_add(1).unwrap();
-                        to_execute_substage.send(ExecutablePayload(ee)).unwrap();
+                        to_high_execute_substage.unwrap_or(to_execute_substage).send(ExecutablePayload(ee)).unwrap();
                     }
                     debug!("schedule_once id_{:016x} [C] ch(prev: {}, exec: {}|{}), r: {}, u/c: {}/{}, (imm+provi)/max: ({}+{})/{} s: {} l(s+f): {}+{}", random_id, (if from_disconnected { "-".to_string() } else { format!("{}", from_prev.len()) }), to_execute_substage.len(), from_exec.len(), runnable_queue.task_count(), address_book.uncontended_task_ids.len(), contended_count, executing_queue_count, provisioning_tracker_count, max_executing_queue_count, address_book.stuck_tasks.len(), processed_count, failed_lock_count);
 
@@ -1628,7 +1625,7 @@ impl ScheduleStage {
                     }
                 }
                 let mut selection = TaskSelection::OnlyFromRunnable;
-                while !runnable_queue.has_no_task() && selection.should_proceed() && executing_queue_count + provisioning_tracker_count < max_executing_queue_count {
+                while !runnable_queue.has_no_task() && selection.should_proceed() && (to_high_execute_substage.is_some() || executing_queue_count + provisioning_tracker_count < max_executing_queue_count) {
                     let prefer_immediate = true; //provisioning_tracker_count / 4 > executing_queue_count;
 
                     let maybe_ee = Self::schedule_next_execution(

From 7c345d686bd6711ac783f9734105e60d912cf275 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 19 Sep 2022 21:51:47 +0900
Subject: [PATCH 2872/3199] save

---
 scheduler/src/lib.rs | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 481ee2fdcd8e1a..34537e5f1e2e0b 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1606,7 +1606,7 @@ impl ScheduleStage {
                         executing_queue_count = executing_queue_count.checked_add(1).unwrap();
                         to_high_execute_substage.unwrap_or(to_execute_substage).send(ExecutablePayload(ee)).unwrap();
                     }
-                    debug!("schedule_once id_{:016x} [C] ch(prev: {}, exec: {}|{}), r: {}, u/c: {}/{}, (imm+provi)/max: ({}+{})/{} s: {} l(s+f): {}+{}", random_id, (if from_disconnected { "-".to_string() } else { format!("{}", from_prev.len()) }), to_execute_substage.len(), from_exec.len(), runnable_queue.task_count(), address_book.uncontended_task_ids.len(), contended_count, executing_queue_count, provisioning_tracker_count, max_executing_queue_count, address_book.stuck_tasks.len(), processed_count, failed_lock_count);
+                    debug!("schedule_once id_{:016x} [C] ch(prev: {}, exec: {}+{}|{}), r: {}, u/c: {}/{}, (imm+provi)/max: ({}+{})/{} s: {} l(s+f): {}+{}", random_id, (if from_disconnected { "-".to_string() } else { format!("{}", from_prev.len()) }), to_high_execute_substage.map(|t| format!("{}", t.len())).unwrap_or("-"), to_execute_substage.len(), from_exec.len(), runnable_queue.task_count(), address_book.uncontended_task_ids.len(), contended_count, executing_queue_count, provisioning_tracker_count, max_executing_queue_count, address_book.stuck_tasks.len(), processed_count, failed_lock_count);
 
                     if no_aggresive_contended && !from_exec.is_empty() {
                         trace!("abort aggressive contended queue processing due to non-empty from_exec");
@@ -1619,7 +1619,7 @@ impl ScheduleStage {
                         if elapsed > std::time::Duration::from_millis(150) {
                             let delta = (processed_count - last_processed_count) as u128;
                             let elapsed2 = elapsed.as_micros();
-                            info!("schedule_once:interval id_{:016x} ch(prev: {}, exec: {}|{}), r: {}, u/c: {}/{}, (imm+provi)/max: ({}+{})/{} s: {} l(s+f): {}+{} ({}txs/{}us={}tps)", random_id, (if from_disconnected { "-".to_string() } else { format!("{}", from_prev.len()) }), to_execute_substage.len(), from_exec.len(), runnable_queue.task_count(), address_book.uncontended_task_ids.len(), contended_count, executing_queue_count, provisioning_tracker_count, max_executing_queue_count, address_book.stuck_tasks.len(), processed_count, failed_lock_count, delta, elapsed.as_micros(), 1_000_000_u128*delta/elapsed2);
+                            info!("schedule_once:interval id_{:016x} ch(prev: {}, exec: {}+{}|{}), r: {}, u/c: {}/{}, (imm+provi)/max: ({}+{})/{} s: {} l(s+f): {}+{} ({}txs/{}us={}tps)", random_id, (if from_disconnected { "-".to_string() } else { format!("{}", from_prev.len()) }), to_high_execute_substage.map(|t| format!("{}", t.len())).unwrap_or("-"), to_execute_substage.len(), from_exec.len(), runnable_queue.task_count(), address_book.uncontended_task_ids.len(), contended_count, executing_queue_count, provisioning_tracker_count, max_executing_queue_count, address_book.stuck_tasks.len(), processed_count, failed_lock_count, delta, elapsed.as_micros(), 1_000_000_u128*delta/elapsed2);
                             (last_time, last_processed_count) = (std::time::Instant::now(), processed_count);
                         }
                     }
@@ -1646,7 +1646,7 @@ impl ScheduleStage {
                         executing_queue_count = executing_queue_count.checked_add(1).unwrap();
                         to_execute_substage.send(ExecutablePayload(ee)).unwrap();
                     }
-                    debug!("schedule_once id_{:016x} [R] ch(prev: {}, exec: {}|{}), r: {}, u/c: {}/{}, (imm+provi)/max: ({}+{})/{} s: {} l(s+f): {}+{}", random_id, (if from_disconnected { "-".to_string() } else { format!("{}", from_prev.len()) }), to_execute_substage.len(), from_exec.len(), runnable_queue.task_count(), address_book.uncontended_task_ids.len(), contended_count, executing_queue_count, provisioning_tracker_count, max_executing_queue_count, address_book.stuck_tasks.len(), processed_count, failed_lock_count);
+                    debug!("schedule_once id_{:016x} [R] ch(prev: {}, exec: {}+{}|{}), r: {}, u/c: {}/{}, (imm+provi)/max: ({}+{})/{} s: {} l(s+f): {}+{}", random_id, (if from_disconnected { "-".to_string() } else { format!("{}", from_prev.len()) }), to_high_execute_substage.map(|t| format!("{}", t.len())).unwrap_or("-"), to_execute_substage.len(), from_exec.len(), runnable_queue.task_count(), address_book.uncontended_task_ids.len(), contended_count, executing_queue_count, provisioning_tracker_count, max_executing_queue_count, address_book.stuck_tasks.len(), processed_count, failed_lock_count);
 
                     if !from_exec.is_empty() {
                         trace!("abort aggressive runnable queue processing due to non-empty from_exec");
@@ -1659,7 +1659,7 @@ impl ScheduleStage {
                         if elapsed > std::time::Duration::from_millis(150) {
                             let delta = (processed_count - last_processed_count) as u128;
                             let elapsed2 = elapsed.as_micros();
-                            info!("schedule_once:interval id_{:016x} ch(prev: {}, exec: {}|{}), r: {}, u/c: {}/{}, (imm+provi)/max: ({}+{})/{} s: {} l(s+f): {}+{} ({}txs/{}us={}tps)", random_id, (if from_disconnected { "-".to_string() } else { format!("{}", from_prev.len()) }), to_execute_substage.len(), from_exec.len(), runnable_queue.task_count(), address_book.uncontended_task_ids.len(), contended_count, executing_queue_count, provisioning_tracker_count, max_executing_queue_count, address_book.stuck_tasks.len(), processed_count, failed_lock_count, delta, elapsed.as_micros(), 1_000_000_u128*delta/elapsed2);
+                            info!("schedule_once:interval id_{:016x} ch(prev: {}, exec: {}+{}|{}), r: {}, u/c: {}/{}, (imm+provi)/max: ({}+{})/{} s: {} l(s+f): {}+{} ({}txs/{}us={}tps)", random_id, (if from_disconnected { "-".to_string() } else { format!("{}", from_prev.len()) }), to_high_execute_substage.map(|t| format!("{}", t.len())).unwrap_or("-"), to_execute_substage.len(), from_exec.len(), runnable_queue.task_count(), address_book.uncontended_task_ids.len(), contended_count, executing_queue_count, provisioning_tracker_count, max_executing_queue_count, address_book.stuck_tasks.len(), processed_count, failed_lock_count, delta, elapsed.as_micros(), 1_000_000_u128*delta/elapsed2);
                             (last_time, last_processed_count) = (std::time::Instant::now(), processed_count);
                         }
                     }

From fdaf5dedde586e1dde86b5575b0e3de946625f70 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 19 Sep 2022 21:52:10 +0900
Subject: [PATCH 2873/3199] save

---
 scheduler/src/lib.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 34537e5f1e2e0b..80b0d0d68a2af3 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1606,7 +1606,7 @@ impl ScheduleStage {
                         executing_queue_count = executing_queue_count.checked_add(1).unwrap();
                         to_high_execute_substage.unwrap_or(to_execute_substage).send(ExecutablePayload(ee)).unwrap();
                     }
-                    debug!("schedule_once id_{:016x} [C] ch(prev: {}, exec: {}+{}|{}), r: {}, u/c: {}/{}, (imm+provi)/max: ({}+{})/{} s: {} l(s+f): {}+{}", random_id, (if from_disconnected { "-".to_string() } else { format!("{}", from_prev.len()) }), to_high_execute_substage.map(|t| format!("{}", t.len())).unwrap_or("-"), to_execute_substage.len(), from_exec.len(), runnable_queue.task_count(), address_book.uncontended_task_ids.len(), contended_count, executing_queue_count, provisioning_tracker_count, max_executing_queue_count, address_book.stuck_tasks.len(), processed_count, failed_lock_count);
+                    debug!("schedule_once id_{:016x} [C] ch(prev: {}, exec: {}+{}|{}), r: {}, u/c: {}/{}, (imm+provi)/max: ({}+{})/{} s: {} l(s+f): {}+{}", random_id, (if from_disconnected { "-".to_string() } else { format!("{}", from_prev.len()) }), to_high_execute_substage.map(|t| format!("{}", t.len()).into()).unwrap_or("-"), to_execute_substage.len(), from_exec.len(), runnable_queue.task_count(), address_book.uncontended_task_ids.len(), contended_count, executing_queue_count, provisioning_tracker_count, max_executing_queue_count, address_book.stuck_tasks.len(), processed_count, failed_lock_count);
 
                     if no_aggresive_contended && !from_exec.is_empty() {
                         trace!("abort aggressive contended queue processing due to non-empty from_exec");
@@ -1619,7 +1619,7 @@ impl ScheduleStage {
                         if elapsed > std::time::Duration::from_millis(150) {
                             let delta = (processed_count - last_processed_count) as u128;
                             let elapsed2 = elapsed.as_micros();
-                            info!("schedule_once:interval id_{:016x} ch(prev: {}, exec: {}+{}|{}), r: {}, u/c: {}/{}, (imm+provi)/max: ({}+{})/{} s: {} l(s+f): {}+{} ({}txs/{}us={}tps)", random_id, (if from_disconnected { "-".to_string() } else { format!("{}", from_prev.len()) }), to_high_execute_substage.map(|t| format!("{}", t.len())).unwrap_or("-"), to_execute_substage.len(), from_exec.len(), runnable_queue.task_count(), address_book.uncontended_task_ids.len(), contended_count, executing_queue_count, provisioning_tracker_count, max_executing_queue_count, address_book.stuck_tasks.len(), processed_count, failed_lock_count, delta, elapsed.as_micros(), 1_000_000_u128*delta/elapsed2);
+                            info!("schedule_once:interval id_{:016x} ch(prev: {}, exec: {}+{}|{}), r: {}, u/c: {}/{}, (imm+provi)/max: ({}+{})/{} s: {} l(s+f): {}+{} ({}txs/{}us={}tps)", random_id, (if from_disconnected { "-".to_string() } else { format!("{}", from_prev.len()) }), to_high_execute_substage.map(|t| format!("{}", t.len()).into()).unwrap_or("-"), to_execute_substage.len(), from_exec.len(), runnable_queue.task_count(), address_book.uncontended_task_ids.len(), contended_count, executing_queue_count, provisioning_tracker_count, max_executing_queue_count, address_book.stuck_tasks.len(), processed_count, failed_lock_count, delta, elapsed.as_micros(), 1_000_000_u128*delta/elapsed2);
                             (last_time, last_processed_count) = (std::time::Instant::now(), processed_count);
                         }
                     }

From cc403ec84b8cad19a0363922125bf6bf13ca8058 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 19 Sep 2022 21:52:48 +0900
Subject: [PATCH 2874/3199] save

---
 scheduler/src/lib.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 80b0d0d68a2af3..135dd7b5c9c597 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1606,7 +1606,7 @@ impl ScheduleStage {
                         executing_queue_count = executing_queue_count.checked_add(1).unwrap();
                         to_high_execute_substage.unwrap_or(to_execute_substage).send(ExecutablePayload(ee)).unwrap();
                     }
-                    debug!("schedule_once id_{:016x} [C] ch(prev: {}, exec: {}+{}|{}), r: {}, u/c: {}/{}, (imm+provi)/max: ({}+{})/{} s: {} l(s+f): {}+{}", random_id, (if from_disconnected { "-".to_string() } else { format!("{}", from_prev.len()) }), to_high_execute_substage.map(|t| format!("{}", t.len()).into()).unwrap_or("-"), to_execute_substage.len(), from_exec.len(), runnable_queue.task_count(), address_book.uncontended_task_ids.len(), contended_count, executing_queue_count, provisioning_tracker_count, max_executing_queue_count, address_book.stuck_tasks.len(), processed_count, failed_lock_count);
+                    debug!("schedule_once id_{:016x} [C] ch(prev: {}, exec: {}+{}|{}), r: {}, u/c: {}/{}, (imm+provi)/max: ({}+{})/{} s: {} l(s+f): {}+{}", random_id, (if from_disconnected { "-".to_string() } else { format!("{}", from_prev.len()) }), to_high_execute_substage.map(|t| format!("{}", t.len())).unwrap_or("-".into()), to_execute_substage.len(), from_exec.len(), runnable_queue.task_count(), address_book.uncontended_task_ids.len(), contended_count, executing_queue_count, provisioning_tracker_count, max_executing_queue_count, address_book.stuck_tasks.len(), processed_count, failed_lock_count);
 
                     if no_aggresive_contended && !from_exec.is_empty() {
                         trace!("abort aggressive contended queue processing due to non-empty from_exec");
@@ -1619,7 +1619,7 @@ impl ScheduleStage {
                         if elapsed > std::time::Duration::from_millis(150) {
                             let delta = (processed_count - last_processed_count) as u128;
                             let elapsed2 = elapsed.as_micros();
-                            info!("schedule_once:interval id_{:016x} ch(prev: {}, exec: {}+{}|{}), r: {}, u/c: {}/{}, (imm+provi)/max: ({}+{})/{} s: {} l(s+f): {}+{} ({}txs/{}us={}tps)", random_id, (if from_disconnected { "-".to_string() } else { format!("{}", from_prev.len()) }), to_high_execute_substage.map(|t| format!("{}", t.len()).into()).unwrap_or("-"), to_execute_substage.len(), from_exec.len(), runnable_queue.task_count(), address_book.uncontended_task_ids.len(), contended_count, executing_queue_count, provisioning_tracker_count, max_executing_queue_count, address_book.stuck_tasks.len(), processed_count, failed_lock_count, delta, elapsed.as_micros(), 1_000_000_u128*delta/elapsed2);
+                            info!("schedule_once:interval id_{:016x} ch(prev: {}, exec: {}+{}|{}), r: {}, u/c: {}/{}, (imm+provi)/max: ({}+{})/{} s: {} l(s+f): {}+{} ({}txs/{}us={}tps)", random_id, (if from_disconnected { "-".to_string() } else { format!("{}", from_prev.len()) }), to_high_execute_substage.map(|t| format!("{}", t.len())).unwrap_or("-"), to_execute_substage.len(), from_exec.len(), runnable_queue.task_count(), address_book.uncontended_task_ids.len(), contended_count, executing_queue_count, provisioning_tracker_count, max_executing_queue_count, address_book.stuck_tasks.len(), processed_count, failed_lock_count, delta, elapsed.as_micros(), 1_000_000_u128*delta/elapsed2);
                             (last_time, last_processed_count) = (std::time::Instant::now(), processed_count);
                         }
                     }

From 6d2a94211dcf92c625f2921ec8131c79515523c5 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 19 Sep 2022 21:53:12 +0900
Subject: [PATCH 2875/3199] save

---
 scheduler/src/lib.rs | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 135dd7b5c9c597..bdc3bf64f479ce 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1619,7 +1619,7 @@ impl ScheduleStage {
                         if elapsed > std::time::Duration::from_millis(150) {
                             let delta = (processed_count - last_processed_count) as u128;
                             let elapsed2 = elapsed.as_micros();
-                            info!("schedule_once:interval id_{:016x} ch(prev: {}, exec: {}+{}|{}), r: {}, u/c: {}/{}, (imm+provi)/max: ({}+{})/{} s: {} l(s+f): {}+{} ({}txs/{}us={}tps)", random_id, (if from_disconnected { "-".to_string() } else { format!("{}", from_prev.len()) }), to_high_execute_substage.map(|t| format!("{}", t.len())).unwrap_or("-"), to_execute_substage.len(), from_exec.len(), runnable_queue.task_count(), address_book.uncontended_task_ids.len(), contended_count, executing_queue_count, provisioning_tracker_count, max_executing_queue_count, address_book.stuck_tasks.len(), processed_count, failed_lock_count, delta, elapsed.as_micros(), 1_000_000_u128*delta/elapsed2);
+                            info!("schedule_once:interval id_{:016x} ch(prev: {}, exec: {}+{}|{}), r: {}, u/c: {}/{}, (imm+provi)/max: ({}+{})/{} s: {} l(s+f): {}+{} ({}txs/{}us={}tps)", random_id, (if from_disconnected { "-".to_string() } else { format!("{}", from_prev.len()) }), to_high_execute_substage.map(|t| format!("{}", t.len())).unwrap_or("-".into()), to_execute_substage.len(), from_exec.len(), runnable_queue.task_count(), address_book.uncontended_task_ids.len(), contended_count, executing_queue_count, provisioning_tracker_count, max_executing_queue_count, address_book.stuck_tasks.len(), processed_count, failed_lock_count, delta, elapsed.as_micros(), 1_000_000_u128*delta/elapsed2);
                             (last_time, last_processed_count) = (std::time::Instant::now(), processed_count);
                         }
                     }
@@ -1646,7 +1646,7 @@ impl ScheduleStage {
                         executing_queue_count = executing_queue_count.checked_add(1).unwrap();
                         to_execute_substage.send(ExecutablePayload(ee)).unwrap();
                     }
-                    debug!("schedule_once id_{:016x} [R] ch(prev: {}, exec: {}+{}|{}), r: {}, u/c: {}/{}, (imm+provi)/max: ({}+{})/{} s: {} l(s+f): {}+{}", random_id, (if from_disconnected { "-".to_string() } else { format!("{}", from_prev.len()) }), to_high_execute_substage.map(|t| format!("{}", t.len())).unwrap_or("-"), to_execute_substage.len(), from_exec.len(), runnable_queue.task_count(), address_book.uncontended_task_ids.len(), contended_count, executing_queue_count, provisioning_tracker_count, max_executing_queue_count, address_book.stuck_tasks.len(), processed_count, failed_lock_count);
+                    debug!("schedule_once id_{:016x} [R] ch(prev: {}, exec: {}+{}|{}), r: {}, u/c: {}/{}, (imm+provi)/max: ({}+{})/{} s: {} l(s+f): {}+{}", random_id, (if from_disconnected { "-".to_string() } else { format!("{}", from_prev.len()) }), to_high_execute_substage.map(|t| format!("{}", t.len())).unwrap_or("-".into()), to_execute_substage.len(), from_exec.len(), runnable_queue.task_count(), address_book.uncontended_task_ids.len(), contended_count, executing_queue_count, provisioning_tracker_count, max_executing_queue_count, address_book.stuck_tasks.len(), processed_count, failed_lock_count);
 
                     if !from_exec.is_empty() {
                         trace!("abort aggressive runnable queue processing due to non-empty from_exec");
@@ -1659,7 +1659,7 @@ impl ScheduleStage {
                         if elapsed > std::time::Duration::from_millis(150) {
                             let delta = (processed_count - last_processed_count) as u128;
                             let elapsed2 = elapsed.as_micros();
-                            info!("schedule_once:interval id_{:016x} ch(prev: {}, exec: {}+{}|{}), r: {}, u/c: {}/{}, (imm+provi)/max: ({}+{})/{} s: {} l(s+f): {}+{} ({}txs/{}us={}tps)", random_id, (if from_disconnected { "-".to_string() } else { format!("{}", from_prev.len()) }), to_high_execute_substage.map(|t| format!("{}", t.len())).unwrap_or("-"), to_execute_substage.len(), from_exec.len(), runnable_queue.task_count(), address_book.uncontended_task_ids.len(), contended_count, executing_queue_count, provisioning_tracker_count, max_executing_queue_count, address_book.stuck_tasks.len(), processed_count, failed_lock_count, delta, elapsed.as_micros(), 1_000_000_u128*delta/elapsed2);
+                            info!("schedule_once:interval id_{:016x} ch(prev: {}, exec: {}+{}|{}), r: {}, u/c: {}/{}, (imm+provi)/max: ({}+{})/{} s: {} l(s+f): {}+{} ({}txs/{}us={}tps)", random_id, (if from_disconnected { "-".to_string() } else { format!("{}", from_prev.len()) }), to_high_execute_substage.map(|t| format!("{}", t.len())).unwrap_or("-".into()), to_execute_substage.len(), from_exec.len(), runnable_queue.task_count(), address_book.uncontended_task_ids.len(), contended_count, executing_queue_count, provisioning_tracker_count, max_executing_queue_count, address_book.stuck_tasks.len(), processed_count, failed_lock_count, delta, elapsed.as_micros(), 1_000_000_u128*delta/elapsed2);
                             (last_time, last_processed_count) = (std::time::Instant::now(), processed_count);
                         }
                     }

From e81a9764c3a6db32d276c66a08b79c06f7af4778 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 19 Sep 2022 22:11:59 +0900
Subject: [PATCH 2876/3199] save

---
 runtime/src/bank.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 5b4c6832500368..611de005d576df 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1307,7 +1307,7 @@ impl Scheduler {
             }
 
 
-            while let Ok(solana_scheduler::ExecutablePayload(mut ee)) = scheduled_high_ee_receiver.try_recv().map_err(|_e| scheduled_ee_receiver.recv()) {
+            while let Ok(solana_scheduler::ExecutablePayload(mut ee)) = scheduled_high_ee_receiver.try_recv().or_else(|_e| scheduled_ee_receiver.recv()) {
                 let (mut wall_time, cpu_time) = (Measure::start("process_message_time"), cpu_time::ThreadTime::now());
 
                 let current_execute_clock = ee.task.execute_time();

From cb4b56d61f8dc4f71d783b47da300b9be151d2b5 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Mon, 19 Sep 2022 22:38:52 +0900
Subject: [PATCH 2877/3199] save

---
 runtime/src/bank.rs | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 611de005d576df..14b5283aba2c87 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1296,7 +1296,7 @@ impl Scheduler {
 
         let max_thread_priority = std::env::var("MAX_THREAD_PRIORITY").is_ok();
 
-        let executing_thread_handles = (0..executing_thread_count).map(|thx| {
+        let executing_thread_handles = (0..(executing_thread_count * 2)).map(|thx| {
             let (scheduled_ee_receiver, scheduled_high_ee_receiver, processed_ee_sender) = (scheduled_ee_receiver.clone(), scheduled_high_ee_receiver.clone(), processed_ee_sender.clone());
             let bank = bank.clone();
 
@@ -1306,8 +1306,7 @@ impl Scheduler {
                 thread_priority::set_current_thread_priority(thread_priority::ThreadPriority::Max).unwrap();
             }
 
-
-            while let Ok(solana_scheduler::ExecutablePayload(mut ee)) = scheduled_high_ee_receiver.try_recv().or_else(|_e| scheduled_ee_receiver.recv()) {
+            while let Ok(solana_scheduler::ExecutablePayload(mut ee)) = (if thx >= executing_thread_count { scheduled_high_ee_receiver.recv() } else { scheduled_ee_receiver.recv()}) {
                 let (mut wall_time, cpu_time) = (Measure::start("process_message_time"), cpu_time::ThreadTime::now());
 
                 let current_execute_clock = ee.task.execute_time();

From d3a4e198979094be99e3e12359658bace94ae27e Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 20 Sep 2022 09:57:40 +0900
Subject: [PATCH 2878/3199] save

---
 runtime/src/bank.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 14b5283aba2c87..bb42c9cf32184c 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1380,7 +1380,7 @@ impl Scheduler {
         let collected_errors = Arc::new(std::sync::Mutex::new(Vec::new()));
         let collected_errors_in_collector_thread = Arc::clone(&collected_errors);
 
-        let error_collector_thread_handle = std::thread::Builder::new().name(format!("solScErrCol{:02}", 0)).spawn(move || {
+        let error_collector_thread_handle = std::thread::Builder::new().name(format!("solScErrCol{:02}", 0)).spawn(fn error_collector_main() { move || {
             let started = (cpu_time::ThreadTime::now(), std::time::Instant::now());
             if max_thread_priority {
                 thread_priority::set_current_thread_priority(thread_priority::ThreadPriority::Max).unwrap();
@@ -1418,7 +1418,7 @@ impl Scheduler {
             todo!();
 
             Ok((started.0.elapsed(), started.1.elapsed()))
-        }).unwrap();
+        }}).unwrap();
 
         let random_id = rand::thread_rng().gen::<u64>();
 

From f4009949b13fe5dc6ffa25c2a92b290e37e22bf6 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 20 Sep 2022 09:58:26 +0900
Subject: [PATCH 2879/3199] save

---
 runtime/src/bank.rs | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index bb42c9cf32184c..10d809d1fbbf94 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1379,8 +1379,7 @@ impl Scheduler {
 
         let collected_errors = Arc::new(std::sync::Mutex::new(Vec::new()));
         let collected_errors_in_collector_thread = Arc::clone(&collected_errors);
-
-        let error_collector_thread_handle = std::thread::Builder::new().name(format!("solScErrCol{:02}", 0)).spawn(fn error_collector_main() { move || {
+        fn error_collector_main() { move || {
             let started = (cpu_time::ThreadTime::now(), std::time::Instant::now());
             if max_thread_priority {
                 thread_priority::set_current_thread_priority(thread_priority::ThreadPriority::Max).unwrap();
@@ -1418,7 +1417,8 @@ impl Scheduler {
             todo!();
 
             Ok((started.0.elapsed(), started.1.elapsed()))
-        }}).unwrap();
+        }}
+        let error_collector_thread_handle = std::thread::Builder::new().name(format!("solScErrCol{:02}", 0)).spawn(error_collector_main).unwrap();
 
         let random_id = rand::thread_rng().gen::<u64>();
 

From e831013b592a48a23c459fc87b026b754c5f06d2 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 20 Sep 2022 10:32:40 +0900
Subject: [PATCH 2880/3199] save

---
 runtime/src/accounts.rs | 1 +
 runtime/src/bank.rs     | 6 +++---
 2 files changed, 4 insertions(+), 3 deletions(-)

diff --git a/runtime/src/accounts.rs b/runtime/src/accounts.rs
index ade9d327ba1046..0670efbe824ebe 100644
--- a/runtime/src/accounts.rs
+++ b/runtime/src/accounts.rs
@@ -525,6 +525,7 @@ impl Accounts {
     }
 
     #[allow(clippy::too_many_arguments)]
+    #[inline(never)]
     pub fn load_accounts(
         &self,
         ancestors: &Ancestors,
diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 10d809d1fbbf94..14b5283aba2c87 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1379,7 +1379,8 @@ impl Scheduler {
 
         let collected_errors = Arc::new(std::sync::Mutex::new(Vec::new()));
         let collected_errors_in_collector_thread = Arc::clone(&collected_errors);
-        fn error_collector_main() { move || {
+
+        let error_collector_thread_handle = std::thread::Builder::new().name(format!("solScErrCol{:02}", 0)).spawn(move || {
             let started = (cpu_time::ThreadTime::now(), std::time::Instant::now());
             if max_thread_priority {
                 thread_priority::set_current_thread_priority(thread_priority::ThreadPriority::Max).unwrap();
@@ -1417,8 +1418,7 @@ impl Scheduler {
             todo!();
 
             Ok((started.0.elapsed(), started.1.elapsed()))
-        }}
-        let error_collector_thread_handle = std::thread::Builder::new().name(format!("solScErrCol{:02}", 0)).spawn(error_collector_main).unwrap();
+        }).unwrap();
 
         let random_id = rand::thread_rng().gen::<u64>();
 

From 6b9425543d5a5ae426f45e45c937238b83d36d07 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 20 Sep 2022 16:55:31 +0900
Subject: [PATCH 2881/3199] save

---
 core/src/replay_stage.rs           |  2 +-
 ledger/src/blockstore_processor.rs | 11 +++++++++++
 2 files changed, 12 insertions(+), 1 deletion(-)

diff --git a/core/src/replay_stage.rs b/core/src/replay_stage.rs
index 4455c9fae5f27c..4a2b48747fac2d 100644
--- a/core/src/replay_stage.rs
+++ b/core/src/replay_stage.rs
@@ -916,7 +916,7 @@ impl ReplayStage {
                     if !did_complete_bank {
                         // only wait for the signal if we did not just process a bank; maybe there are more slots available
 
-                        let timer = Duration::from_millis(10);
+                        let timer = Duration::from_millis(2000);
                         let result = ledger_signal_receiver.recv_timeout(timer);
                         match result {
                             Err(RecvTimeoutError::Timeout) => (),
diff --git a/ledger/src/blockstore_processor.rs b/ledger/src/blockstore_processor.rs
index a94ebbd7453def..a1fbd6545d8e26 100644
--- a/ledger/src/blockstore_processor.rs
+++ b/ledger/src/blockstore_processor.rs
@@ -186,9 +186,11 @@ fn execute_batch(
         transactions,
         transaction_indexes,
     );
+    /*
     if let Some(first_error_from_scheduler) = bank.handle_aborted_transactions().into_iter().next() {
         first_error_from_scheduler?
     }
+    */
     return Ok(());
 
     /*
@@ -417,6 +419,8 @@ fn process_entries_with_callback(
 ) -> Result<()> {
     // accumulator for entries that can be processed in parallel
     let mut tick_hashes = vec![];
+    let mut started = std::time::Instant::now();
+    let mut tx_count = 0;
 
     for ReplayEntry {
         entry,
@@ -428,6 +432,10 @@ fn process_entries_with_callback(
                 // If it's a tick, save it for later
                 tick_hashes.push(hash);
                 if bank.is_block_boundary(bank.tick_height() + tick_hashes.len() as u64) {
+                    info!("exeucted {} txs so far in {}us", tx_count, started.elapsed().as_micros());
+                    started = std::time::Instant::now();
+                    tx_count = 0;
+
                     for hash in &tick_hashes {
                         bank.register_tick(hash);
                     }
@@ -437,6 +445,7 @@ fn process_entries_with_callback(
             EntryType::Transactions(transactions) => {
                 let starting_index = *starting_index;
                 let transaction_indexes = (starting_index..starting_index.saturating_add(transactions.len())).into_iter();
+                tx_count += transactions.len();
 
                 execute_batches(
                     bank,
@@ -452,6 +461,8 @@ fn process_entries_with_callback(
             }
         }
     }
+    info!("exeucted {} txs so far in {}us", tx_count, started.elapsed().as_micros());
+    started = std::time::Instant::now();
     for hash in tick_hashes {
         bank.register_tick(hash);
     }

From 3201d8b9b6b2badb9041c049acd4225572819c88 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 20 Sep 2022 17:00:28 +0900
Subject: [PATCH 2882/3199] save

---
 scheduler/src/lib.rs | 25 ++++++++++++++++---------
 1 file changed, 16 insertions(+), 9 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index bdc3bf64f479ce..fca44655635fa2 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1443,8 +1443,8 @@ impl ScheduleStage {
         maybe_to_next_stage: Option<&crossbeam_channel::Sender<ExaminablePayload>>, // assume nonblocking
         never: &'a crossbeam_channel::Receiver<SchedulablePayload>,
     ) -> Option<std::sync::Arc<Checkpoint>> {
-        let start_time = std::time::Instant::now();
-        let (mut last_time, mut last_processed_count) = (start_time.clone(), 0_usize);
+        let maybe_start_time = None;
+        let (mut last_time, mut last_processed_count) = (maybe_start_time.clone(), 0_usize);
         info!("schedule_once:initial id_{:016x}", random_id);
 
         let mut executing_queue_count = 0_usize;
@@ -1543,6 +1543,10 @@ impl ScheduleStage {
                        }
                    }
                    recv(from_prev) -> maybe_from => {
+                       if maybe_start_time.is_none() {
+                           maybe_start_time = Some(std::time::Instant::now());
+                           last_time = maybe_start_time.clone();
+                       }
                        match maybe_from {
                            Ok(SchedulablePayload(Flushable::Payload(task))) => {
                                 Self::register_runnable_task(task, runnable_queue, &mut sequence_time);
@@ -1615,12 +1619,12 @@ impl ScheduleStage {
 
                     interval_count += 1;
                     if interval_count % 100 == 0 {
-                        let elapsed = last_time.elapsed();
+                        let elapsed = last_time.unwrap().elapsed();
                         if elapsed > std::time::Duration::from_millis(150) {
                             let delta = (processed_count - last_processed_count) as u128;
                             let elapsed2 = elapsed.as_micros();
                             info!("schedule_once:interval id_{:016x} ch(prev: {}, exec: {}+{}|{}), r: {}, u/c: {}/{}, (imm+provi)/max: ({}+{})/{} s: {} l(s+f): {}+{} ({}txs/{}us={}tps)", random_id, (if from_disconnected { "-".to_string() } else { format!("{}", from_prev.len()) }), to_high_execute_substage.map(|t| format!("{}", t.len())).unwrap_or("-".into()), to_execute_substage.len(), from_exec.len(), runnable_queue.task_count(), address_book.uncontended_task_ids.len(), contended_count, executing_queue_count, provisioning_tracker_count, max_executing_queue_count, address_book.stuck_tasks.len(), processed_count, failed_lock_count, delta, elapsed.as_micros(), 1_000_000_u128*delta/elapsed2);
-                            (last_time, last_processed_count) = (std::time::Instant::now(), processed_count);
+                            (last_time, last_processed_count) = (Some(std::time::Instant::now()), processed_count);
                         }
                     }
                 }
@@ -1655,12 +1659,12 @@ impl ScheduleStage {
 
                     interval_count += 1;
                     if interval_count % 100 == 0 {
-                        let elapsed = last_time.elapsed();
+                        let elapsed = last_time.unwrap().elapsed();
                         if elapsed > std::time::Duration::from_millis(150) {
                             let delta = (processed_count - last_processed_count) as u128;
                             let elapsed2 = elapsed.as_micros();
                             info!("schedule_once:interval id_{:016x} ch(prev: {}, exec: {}+{}|{}), r: {}, u/c: {}/{}, (imm+provi)/max: ({}+{})/{} s: {} l(s+f): {}+{} ({}txs/{}us={}tps)", random_id, (if from_disconnected { "-".to_string() } else { format!("{}", from_prev.len()) }), to_high_execute_substage.map(|t| format!("{}", t.len())).unwrap_or("-".into()), to_execute_substage.len(), from_exec.len(), runnable_queue.task_count(), address_book.uncontended_task_ids.len(), contended_count, executing_queue_count, provisioning_tracker_count, max_executing_queue_count, address_book.stuck_tasks.len(), processed_count, failed_lock_count, delta, elapsed.as_micros(), 1_000_000_u128*delta/elapsed2);
-                            (last_time, last_processed_count) = (std::time::Instant::now(), processed_count);
+                            (last_time, last_processed_count) = (Some(std::time::Instant::now()), processed_count);
                         }
                     }
                 }
@@ -1734,9 +1738,12 @@ impl ScheduleStage {
         }
 
 
-        let elapsed = start_time.elapsed();
-        let elapsed2 = elapsed.as_micros();
-        info!("schedule_once:final id_{:016x} (no_more_work: {}) ch(prev: {}, exec: {}|{}), runnnable: {}, contended: {}, (immediate+provisional)/max: ({}+{})/{} uncontended: {} stuck: {} miss: {}, overall: {}txs/{}us={}tps!", random_id, no_more_work, (if from_disconnected { "-".to_string() } else { format!("{}", from_prev.len()) }), to_execute_substage.len(), (if from_exec_disconnected { "-".to_string() } else { format!("{}", from_exec.len())}), runnable_queue.task_count(), contended_count, executing_queue_count, provisioning_tracker_count, max_executing_queue_count, address_book.uncontended_task_ids.len(), address_book.stuck_tasks.len(), failed_lock_count, processed_count, elapsed.as_micros(), 1_000_000_u128*(processed_count as u128)/elapsed2);
+        if let Some(start_time) = maybe_start_time {
+            let elapsed = start_time.elapsed();
+            let elapsed2 = elapsed.as_micros();
+            info!("schedule_once:final id_{:016x} (no_more_work: {}) ch(prev: {}, exec: {}|{}), runnnable: {}, contended: {}, (immediate+provisional)/max: ({}+{})/{} uncontended: {} stuck: {} miss: {}, overall: {}txs/{}us={}tps!", random_id, no_more_work, (if from_disconnected { "-".to_string() } else { format!("{}", from_prev.len()) }), to_execute_substage.len(), (if from_exec_disconnected { "-".to_string() } else { format!("{}", from_exec.len())}), runnable_queue.task_count(), contended_count, executing_queue_count, provisioning_tracker_count, max_executing_queue_count, address_book.uncontended_task_ids.len(), address_book.stuck_tasks.len(), failed_lock_count, processed_count, elapsed.as_micros(), 1_000_000_u128*(processed_count as u128)/elapsed2);
+        }
+
         maybe_checkpoint
     }
 

From dc2996e4719e92ef6c7571c840976e5bf6fb764d Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 20 Sep 2022 17:00:39 +0900
Subject: [PATCH 2883/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index fca44655635fa2..bfcf98f6e9e7df 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1443,7 +1443,7 @@ impl ScheduleStage {
         maybe_to_next_stage: Option<&crossbeam_channel::Sender<ExaminablePayload>>, // assume nonblocking
         never: &'a crossbeam_channel::Receiver<SchedulablePayload>,
     ) -> Option<std::sync::Arc<Checkpoint>> {
-        let maybe_start_time = None;
+        let mut maybe_start_time = None;
         let (mut last_time, mut last_processed_count) = (maybe_start_time.clone(), 0_usize);
         info!("schedule_once:initial id_{:016x}", random_id);
 

From ba0484f4c4a130fa4f2239ccb5cf9007b7c7a4f2 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 20 Sep 2022 17:01:23 +0900
Subject: [PATCH 2884/3199] save

---
 scheduler/src/lib.rs | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index bfcf98f6e9e7df..c26922a8358cdf 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1445,7 +1445,7 @@ impl ScheduleStage {
     ) -> Option<std::sync::Arc<Checkpoint>> {
         let mut maybe_start_time = None;
         let (mut last_time, mut last_processed_count) = (maybe_start_time.clone(), 0_usize);
-        info!("schedule_once:initial id_{:016x}", random_id);
+        info!("schedule_once:stanby id_{:016x}", random_id);
 
         let mut executing_queue_count = 0_usize;
         let mut contended_count = 0;
@@ -1544,6 +1544,7 @@ impl ScheduleStage {
                    }
                    recv(from_prev) -> maybe_from => {
                        if maybe_start_time.is_none() {
+                            info!("schedule_once:initial id_{:016x}", random_id);
                            maybe_start_time = Some(std::time::Instant::now());
                            last_time = maybe_start_time.clone();
                        }

From 0be4e6ce087c5ca21e8cde4af01fce99db2200a2 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Tue, 20 Sep 2022 17:02:24 +0900
Subject: [PATCH 2885/3199] save

---
 ledger/src/blockstore_processor.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/ledger/src/blockstore_processor.rs b/ledger/src/blockstore_processor.rs
index a1fbd6545d8e26..2581b09ab1bfa1 100644
--- a/ledger/src/blockstore_processor.rs
+++ b/ledger/src/blockstore_processor.rs
@@ -432,7 +432,7 @@ fn process_entries_with_callback(
                 // If it's a tick, save it for later
                 tick_hashes.push(hash);
                 if bank.is_block_boundary(bank.tick_height() + tick_hashes.len() as u64) {
-                    info!("exeucted {} txs so far in {}us", tx_count, started.elapsed().as_micros());
+                    info!("process_entries_with_callback(): exeucted {} txs so far in {}us (still in loop)", tx_count, started.elapsed().as_micros());
                     started = std::time::Instant::now();
                     tx_count = 0;
 
@@ -461,7 +461,7 @@ fn process_entries_with_callback(
             }
         }
     }
-    info!("exeucted {} txs so far in {}us", tx_count, started.elapsed().as_micros());
+    info!("process_entries_with_callback(): exeucted {} txs so far in {}us (after loop)", tx_count, started.elapsed().as_micros());
     started = std::time::Instant::now();
     for hash in tick_hashes {
         bank.register_tick(hash);

From 81eb94c9be16b8dc8b976f141913417a34db6dd4 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Wed, 21 Sep 2022 00:01:45 +0900
Subject: [PATCH 2886/3199] save

---
 scheduler/src/lib.rs | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index c26922a8358cdf..e02e72d4656749 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1742,7 +1742,12 @@ impl ScheduleStage {
         if let Some(start_time) = maybe_start_time {
             let elapsed = start_time.elapsed();
             let elapsed2 = elapsed.as_micros();
-            info!("schedule_once:final id_{:016x} (no_more_work: {}) ch(prev: {}, exec: {}|{}), runnnable: {}, contended: {}, (immediate+provisional)/max: ({}+{})/{} uncontended: {} stuck: {} miss: {}, overall: {}txs/{}us={}tps!", random_id, no_more_work, (if from_disconnected { "-".to_string() } else { format!("{}", from_prev.len()) }), to_execute_substage.len(), (if from_exec_disconnected { "-".to_string() } else { format!("{}", from_exec.len())}), runnable_queue.task_count(), contended_count, executing_queue_count, provisioning_tracker_count, max_executing_queue_count, address_book.uncontended_task_ids.len(), address_book.stuck_tasks.len(), failed_lock_count, processed_count, elapsed.as_micros(), 1_000_000_u128*(processed_count as u128)/elapsed2);
+            let tps_label = if elapsed2 > 0 {
+                format!("{}", 1_000_000_u128*(processed_count as u128)/elapsed2)
+            } else {
+                "-"
+            };
+            info!("schedule_once:final id_{:016x} (no_more_work: {}) ch(prev: {}, exec: {}|{}), runnnable: {}, contended: {}, (immediate+provisional)/max: ({}+{})/{} uncontended: {} stuck: {} miss: {}, overall: {}txs/{}us={}tps!", random_id, no_more_work, (if from_disconnected { "-".to_string() } else { format!("{}", from_prev.len()) }), to_execute_substage.len(), (if from_exec_disconnected { "-".to_string() } else { format!("{}", from_exec.len())}), runnable_queue.task_count(), contended_count, executing_queue_count, provisioning_tracker_count, max_executing_queue_count, address_book.uncontended_task_ids.len(), address_book.stuck_tasks.len(), failed_lock_count, processed_count, elapsed.as_micros(), tps_label);
         }
 
         maybe_checkpoint

From b34684804c93368fe188b97bec057c860d328790 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Wed, 21 Sep 2022 00:01:58 +0900
Subject: [PATCH 2887/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index e02e72d4656749..dc364dbcaf5ce9 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1745,7 +1745,7 @@ impl ScheduleStage {
             let tps_label = if elapsed2 > 0 {
                 format!("{}", 1_000_000_u128*(processed_count as u128)/elapsed2)
             } else {
-                "-"
+                "-".into()
             };
             info!("schedule_once:final id_{:016x} (no_more_work: {}) ch(prev: {}, exec: {}|{}), runnnable: {}, contended: {}, (immediate+provisional)/max: ({}+{})/{} uncontended: {} stuck: {} miss: {}, overall: {}txs/{}us={}tps!", random_id, no_more_work, (if from_disconnected { "-".to_string() } else { format!("{}", from_prev.len()) }), to_execute_substage.len(), (if from_exec_disconnected { "-".to_string() } else { format!("{}", from_exec.len())}), runnable_queue.task_count(), contended_count, executing_queue_count, provisioning_tracker_count, max_executing_queue_count, address_book.uncontended_task_ids.len(), address_book.stuck_tasks.len(), failed_lock_count, processed_count, elapsed.as_micros(), tps_label);
         }

From f45471206de3d26ece264adafba1fd4e1df20850 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Wed, 21 Sep 2022 00:02:59 +0900
Subject: [PATCH 2888/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index dc364dbcaf5ce9..308930bd708dbe 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1445,7 +1445,7 @@ impl ScheduleStage {
     ) -> Option<std::sync::Arc<Checkpoint>> {
         let mut maybe_start_time = None;
         let (mut last_time, mut last_processed_count) = (maybe_start_time.clone(), 0_usize);
-        info!("schedule_once:stanby id_{:016x}", random_id);
+        info!("schedule_once:standby id_{:016x}", random_id);
 
         let mut executing_queue_count = 0_usize;
         let mut contended_count = 0;

From eea9f838d740f0039807efc3d6dd6459eb99f633 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Wed, 21 Sep 2022 00:39:31 +0900
Subject: [PATCH 2889/3199] save

---
 ledger/src/blockstore_processor.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/ledger/src/blockstore_processor.rs b/ledger/src/blockstore_processor.rs
index 2581b09ab1bfa1..033ae4ee826d31 100644
--- a/ledger/src/blockstore_processor.rs
+++ b/ledger/src/blockstore_processor.rs
@@ -432,7 +432,7 @@ fn process_entries_with_callback(
                 // If it's a tick, save it for later
                 tick_hashes.push(hash);
                 if bank.is_block_boundary(bank.tick_height() + tick_hashes.len() as u64) {
-                    info!("process_entries_with_callback(): exeucted {} txs so far in {}us (still in loop)", tx_count, started.elapsed().as_micros());
+                    info!("process_entries_with_callback(): slot: {} idx: {} exeucted {} txs so far in {}us (still in loop)", bank.slot(), starting_index, tx_count, started.elapsed().as_micros());
                     started = std::time::Instant::now();
                     tx_count = 0;
 
@@ -461,7 +461,7 @@ fn process_entries_with_callback(
             }
         }
     }
-    info!("process_entries_with_callback(): exeucted {} txs so far in {}us (after loop)", tx_count, started.elapsed().as_micros());
+    info!("process_entries_with_callback(): slot: {} idx: {} exeucted {} txs so far in {}us (after loop)", bank.slot(), starting_index, tx_count, started.elapsed().as_micros());
     started = std::time::Instant::now();
     for hash in tick_hashes {
         bank.register_tick(hash);

From e9e574c0d988506773322dfeb2bbf6aa00dce1db Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Wed, 21 Sep 2022 00:40:02 +0900
Subject: [PATCH 2890/3199] save

---
 ledger/src/blockstore_processor.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ledger/src/blockstore_processor.rs b/ledger/src/blockstore_processor.rs
index 033ae4ee826d31..2637da3edb312e 100644
--- a/ledger/src/blockstore_processor.rs
+++ b/ledger/src/blockstore_processor.rs
@@ -461,7 +461,7 @@ fn process_entries_with_callback(
             }
         }
     }
-    info!("process_entries_with_callback(): slot: {} idx: {} exeucted {} txs so far in {}us (after loop)", bank.slot(), starting_index, tx_count, started.elapsed().as_micros());
+    info!("process_entries_with_callback(): slot: {} exeucted {} txs so far in {}us (after loop)", bank.slot(), tx_count, started.elapsed().as_micros());
     started = std::time::Instant::now();
     for hash in tick_hashes {
         bank.register_tick(hash);

From f43d0f7d745f72dc9d12846457e32a9a5ab6320b Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Wed, 21 Sep 2022 13:16:54 +0900
Subject: [PATCH 2891/3199] save

---
 core/src/replay_stage.rs                |   2 +-
 core/src/transaction_scheduler.rs       |  25 +-
 ledger-tool/src/main.rs                 |  77 +++---
 ledger/src/blockstore_processor.rs      |  32 +--
 runtime/src/account_rent_state.rs       |   9 +-
 runtime/src/bank.rs                     | 319 ++++++++++++++++--------
 runtime/src/lib.rs                      |   1 -
 scheduler/src/lib.rs                    | 278 ++++++++++++++-------
 transaction-scheduler-bench/src/main.rs | 195 ++++++++-------
 9 files changed, 577 insertions(+), 361 deletions(-)

diff --git a/core/src/replay_stage.rs b/core/src/replay_stage.rs
index 4a2b48747fac2d..e2c9dc46d6df2f 100644
--- a/core/src/replay_stage.rs
+++ b/core/src/replay_stage.rs
@@ -2476,7 +2476,7 @@ impl ReplayStage {
 
             assert_eq!(bank_slot, bank.slot());
             if bank.is_complete() {
-                info!("waiting for completed bank: slot: {}",bank.slot());
+                info!("waiting for completed bank: slot: {}", bank.slot());
                 let r = bank.wait_for_scheduler(false);
                 if let Err(err) = r {
                     // Error means the slot needs to be marked as dead
diff --git a/core/src/transaction_scheduler.rs b/core/src/transaction_scheduler.rs
index 1d99441a4ee871..5adf0f0561bfdc 100644
--- a/core/src/transaction_scheduler.rs
+++ b/core/src/transaction_scheduler.rs
@@ -2,14 +2,15 @@
 
 use {
     crate::{
-        unprocessed_packet_batches::{self},
         immutable_deserialized_packet::ImmutableDeserializedPacket,
+        unprocessed_packet_batches::{self},
     },
     crossbeam_channel::{Receiver, Sender, TryRecvError},
     dashmap::DashMap,
     solana_measure::{measure, measure::Measure},
     solana_perf::packet::PacketBatch,
-    solana_runtime::{bank::Bank, bank_forks::BankForks,
+    solana_runtime::{
+        bank::Bank, bank_forks::BankForks,
         transaction_priority_details::GetTransactionPriorityDetails,
     },
     solana_sdk::{
@@ -206,7 +207,10 @@ impl PacketBatchHandler {
     fn insert_transactions(&self, transactions: Vec<TransactionRef>) {
         for tx in &transactions {
             // Get account locks
-            let account_locks = tx.transaction.get_account_locks(usize::max_value()).unwrap();
+            let account_locks = tx
+                .transaction
+                .get_account_locks(usize::max_value())
+                .unwrap();
             for account in account_locks.readonly.into_iter() {
                 self.transactions_by_account.entry(*account).or_default();
                 // .reads
@@ -652,7 +656,10 @@ impl TransactionScheduler {
     ) -> Option<HashSet<TransactionBatchId>> {
         let mut conflicting_batches = HashSet::default();
 
-        let account_locks = transaction.transaction.get_account_locks(usize::max_value()).unwrap();
+        let account_locks = transaction
+            .transaction
+            .get_account_locks(usize::max_value())
+            .unwrap();
 
         // Read accounts will only be blocked by writes on other threads
         for account in account_locks.readonly.into_iter() {
@@ -692,7 +699,10 @@ impl TransactionScheduler {
         transaction: &TransactionRef,
         batch_id: TransactionBatchId,
     ) {
-        let accounts = transaction.transaction.get_account_locks(usize::max_value()).unwrap();
+        let accounts = transaction
+            .transaction
+            .get_account_locks(usize::max_value())
+            .unwrap();
         for account in accounts.readonly {
             let mut queue = self.transactions_by_account.get_mut(account).unwrap();
             queue.scheduled_lock.lock_on_batch(batch_id, false);
@@ -810,7 +820,10 @@ impl TransactionBatch {
         self.scheduled = true;
         self.num_transactions = self.transactions.len();
         for transaction in self.transactions.iter() {
-            let account_locks = transaction.transaction.get_account_locks(usize::max_value()).unwrap();
+            let account_locks = transaction
+                .transaction
+                .get_account_locks(usize::max_value())
+                .unwrap();
 
             for account in account_locks.readonly.into_iter() {
                 self.account_locks
diff --git a/ledger-tool/src/main.rs b/ledger-tool/src/main.rs
index dc4f91fd2f5701..a31e8a6bfc1066 100644
--- a/ledger-tool/src/main.rs
+++ b/ledger-tool/src/main.rs
@@ -59,7 +59,9 @@ use {
             DEFAULT_MAX_INCREMENTAL_SNAPSHOT_ARCHIVES_TO_RETAIN, SUPPORTED_ARCHIVE_COMPRESSION,
         },
     },
-    solana_scheduler::{AddressBook, ScheduleStage, TaskQueue, Weight, LockAttempt, RequestedUsage, Preloader},
+    solana_scheduler::{
+        AddressBook, LockAttempt, Preloader, RequestedUsage, ScheduleStage, TaskQueue, Weight,
+    },
     solana_sdk::{
         account::{AccountSharedData, ReadableAccount, WritableAccount},
         account_utils::StateMut,
@@ -177,15 +179,15 @@ fn output_entry(
                     || !solana_runtime::vote_parser::is_simple_vote_transaction(&sanitized_tx)
                 {
                     let locks = sanitized_tx.get_account_locks(usize::max_value()).unwrap();
-                    let writable_lock_iter = locks
-                        .writable
-                        .iter()
-                        .map(|address| LockAttempt::new(preloader.load(**address), RequestedUsage::Writable));
-                    let readonly_lock_iter = locks
-                        .readonly
-                        .iter()
-                        .map(|address| LockAttempt::new(preloader.load(**address), RequestedUsage::Readonly));
-                    let locks = writable_lock_iter.chain(readonly_lock_iter).collect::<Vec<_>>();
+                    let writable_lock_iter = locks.writable.iter().map(|address| {
+                        LockAttempt::new(preloader.load(**address), RequestedUsage::Writable)
+                    });
+                    let readonly_lock_iter = locks.readonly.iter().map(|address| {
+                        LockAttempt::new(preloader.load(**address), RequestedUsage::Readonly)
+                    });
+                    let locks = writable_lock_iter
+                        .chain(readonly_lock_iter)
+                        .collect::<Vec<_>>();
                     to_schedule_stage.push(Box::new((sanitized_tx, locks)));
                 }
                 /*
@@ -284,18 +286,19 @@ fn output_slot(
             use rand::Rng;
             let random_id = rand::thread_rng().gen::<u64>();
             loop {
-            ScheduleStage::run(
-                random_id,
-                lane_count * lane_channel_factor,
-                &mut runnable_queue,
-                &mut address_book,
-                &muxed_receiver,
-                &pre_execute_env_sender,
-                None,
-                &post_execute_env_receiver,
-                Some(&post_schedule_env_sender),
-            );
-        }})
+                ScheduleStage::run(
+                    random_id,
+                    lane_count * lane_channel_factor,
+                    &mut runnable_queue,
+                    &mut address_book,
+                    &muxed_receiver,
+                    &pre_execute_env_sender,
+                    None,
+                    &post_execute_env_receiver,
+                    Some(&post_schedule_env_sender),
+                );
+            }
+        })
         .unwrap();
     let handles = (0..lane_count)
         .map(|thx| {
@@ -347,7 +350,9 @@ fn output_slot(
                         */
                         // ee.reindex_with_address_book();
                         todo!("contended_write_task_count");
-                        post_execute_env_sender.send(solana_scheduler::UnlockablePayload(ee)).unwrap();
+                        post_execute_env_sender
+                            .send(solana_scheduler::UnlockablePayload(ee))
+                            .unwrap();
                     }
                 })
                 .unwrap();
@@ -363,15 +368,15 @@ fn output_slot(
         .parse::<usize>()
         .unwrap();
     let step = Arc::new(std::sync::atomic::AtomicUsize::default());
-    let handles3 = (0..consumer_count).map(|thx| {
-        let post_schedule_env_receiver = post_schedule_env_receiver.clone();
-        let d = d.clone();
-        let step = step.clone();
-
-        let t3 = std::thread::Builder::new()
-            .name(format!("sol-consumer{}", thx))
-            .spawn(move || {
-                loop {
+    let handles3 = (0..consumer_count)
+        .map(|thx| {
+            let post_schedule_env_receiver = post_schedule_env_receiver.clone();
+            let d = d.clone();
+            let step = step.clone();
+
+            let t3 = std::thread::Builder::new()
+                .name(format!("sol-consumer{}", thx))
+                .spawn(move || loop {
                     let ee = post_schedule_env_receiver.recv().unwrap().0;
                     d.fetch_sub(1, Ordering::Relaxed);
                     let step = step.fetch_add(1, Ordering::Relaxed);
@@ -383,11 +388,11 @@ fn output_slot(
                     if step % 1966 == 0 {
                         error!("finished!: {} {}", step, post_schedule_env_receiver.len());
                     }
-                }
-            })
-            .unwrap();
+                })
+                .unwrap();
             t3
-    }).collect::<Vec<_>>();
+        })
+        .collect::<Vec<_>>();
 
     if verbose_level >= 2 {
         let mut txes = Vec::new();
diff --git a/ledger/src/blockstore_processor.rs b/ledger/src/blockstore_processor.rs
index 2637da3edb312e..143efdd8e08a47 100644
--- a/ledger/src/blockstore_processor.rs
+++ b/ledger/src/blockstore_processor.rs
@@ -181,11 +181,7 @@ fn execute_batch(
     };
     */
 
-    bank.schedule_and_commit_transactions(
-        bank,
-        transactions,
-        transaction_indexes,
-    );
+    bank.schedule_and_commit_transactions(bank, transactions, transaction_indexes);
     /*
     if let Some(first_error_from_scheduler) = bank.handle_aborted_transactions().into_iter().next() {
         first_error_from_scheduler?
@@ -283,11 +279,7 @@ fn execute_batches_internal(
     entry_callback: Option<&ProcessCallback>,
 ) -> Result<ExecuteBatchesInternalMetrics> {
     let mut execute_batches_elapsed = Measure::start("execute_batches_elapsed");
-    let result = execute_batch(
-        transactions,
-        transaction_indexes,
-        bank,
-    );
+    let result = execute_batch(transactions, transaction_indexes, bank);
     if let Some(entry_callback) = entry_callback {
         entry_callback(bank);
     }
@@ -335,12 +327,8 @@ fn execute_batches(
     cost_capacity_meter: Arc<RwLock<BlockCostCapacityMeter>>,
     log_messages_bytes_limit: Option<usize>,
 ) -> Result<()> {
-    let execute_batches_internal_metrics = execute_batches_internal(
-        bank,
-        transactions,
-        transaction_indexes,
-        entry_callback,
-    )?;
+    let execute_batches_internal_metrics =
+        execute_batches_internal(bank, transactions, transaction_indexes, entry_callback)?;
 
     confirmation_timing.process_execute_batches_internal_metrics(execute_batches_internal_metrics);
     Ok(())
@@ -432,7 +420,7 @@ fn process_entries_with_callback(
                 // If it's a tick, save it for later
                 tick_hashes.push(hash);
                 if bank.is_block_boundary(bank.tick_height() + tick_hashes.len() as u64) {
-                    info!("process_entries_with_callback(): slot: {} idx: {} exeucted {} txs so far in {}us (still in loop)", bank.slot(), starting_index, tx_count, started.elapsed().as_micros());
+                    info!("process_entries_with_callback(): slot: {} idx: {} scheduled {} txs so far in {}us (still in loop)", bank.slot(), starting_index, tx_count, started.elapsed().as_micros());
                     started = std::time::Instant::now();
                     tx_count = 0;
 
@@ -444,7 +432,8 @@ fn process_entries_with_callback(
             }
             EntryType::Transactions(transactions) => {
                 let starting_index = *starting_index;
-                let transaction_indexes = (starting_index..starting_index.saturating_add(transactions.len())).into_iter();
+                let transaction_indexes =
+                    (starting_index..starting_index.saturating_add(transactions.len())).into_iter();
                 tx_count += transactions.len();
 
                 execute_batches(
@@ -461,7 +450,12 @@ fn process_entries_with_callback(
             }
         }
     }
-    info!("process_entries_with_callback(): slot: {} exeucted {} txs so far in {}us (after loop)", bank.slot(), tx_count, started.elapsed().as_micros());
+    info!(
+        "process_entries_with_callback(): slot: {} scheduled {} txs so far in {}us (after loop)",
+        bank.slot(),
+        tx_count,
+        started.elapsed().as_micros()
+    );
     started = std::time::Instant::now();
     for hash in tick_hashes {
         bank.register_tick(hash);
diff --git a/runtime/src/account_rent_state.rs b/runtime/src/account_rent_state.rs
index 1036f18db0c9bb..1d63cd5afdd059 100644
--- a/runtime/src/account_rent_state.rs
+++ b/runtime/src/account_rent_state.rs
@@ -71,12 +71,9 @@ impl RentState {
 
 pub(crate) fn submit_rent_state_metrics(pre_rent_state: &RentState, post_rent_state: &RentState) {
     match (pre_rent_state, post_rent_state) {
-        (&RentState::Uninitialized, &RentState::RentPaying { .. }) => {
-        }
-        (&RentState::RentPaying { .. }, &RentState::RentPaying { .. }) => {
-        }
-        (_, &RentState::RentPaying { .. }) => {
-        }
+        (&RentState::Uninitialized, &RentState::RentPaying { .. }) => {}
+        (&RentState::RentPaying { .. }, &RentState::RentPaying { .. }) => {}
+        (_, &RentState::RentPaying { .. }) => {}
         _ => {}
     }
 }
diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 14b5283aba2c87..c3d680a6069350 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1212,7 +1212,12 @@ impl SchedulerPool {
 
     fn take_from_pool(&mut self) -> Arc<Scheduler> {
         if let Some(scheduler) = self.schedulers.pop() {
-            info!("SchedulerPool: id_{:016x} is taken... len: {} => {}", scheduler.random_id, self.schedulers.len() + 1, self.schedulers.len());
+            info!(
+                "SchedulerPool: id_{:016x} is taken... len: {} => {}",
+                scheduler.random_id,
+                self.schedulers.len() + 1,
+                self.schedulers.len()
+            );
             scheduler
         } else {
             self.create();
@@ -1221,18 +1226,28 @@ impl SchedulerPool {
     }
 
     fn return_to_pool(&mut self, scheduler: Arc<Scheduler>) {
-        info!("SchedulerPool: id_{:016x} is returned... len: {} => {}", scheduler.random_id, self.schedulers.len(), self.schedulers.len() + 1);
+        info!(
+            "SchedulerPool: id_{:016x} is returned... len: {} => {}",
+            scheduler.random_id,
+            self.schedulers.len(),
+            self.schedulers.len() + 1
+        );
         assert_eq!(1, Arc::strong_count(&scheduler));
         assert!(scheduler.collected_errors.lock().unwrap().is_empty());
-        assert!(scheduler.graceful_stop_initiated.load(std::sync::atomic::Ordering::SeqCst));
+        assert!(scheduler
+            .graceful_stop_initiated
+            .load(std::sync::atomic::Ordering::SeqCst));
 
-        scheduler.graceful_stop_initiated.store(false, std::sync::atomic::Ordering::SeqCst);
+        scheduler
+            .graceful_stop_initiated
+            .store(false, std::sync::atomic::Ordering::SeqCst);
 
         self.schedulers.push(scheduler);
     }
 }
 
-static SCHEDULER_POOL: std::sync::Mutex<SchedulerPool> = std::sync::Mutex::new(SchedulerPool::new());
+static SCHEDULER_POOL: std::sync::Mutex<SchedulerPool> =
+    std::sync::Mutex::new(SchedulerPool::new());
 
 #[derive(Debug)]
 struct Scheduler {
@@ -1257,20 +1272,33 @@ impl Scheduler {
         let nast = NotAtTopOfScheduleThread;
 
         let locks = sanitized_tx.get_account_locks_unchecked();
-        let writable_lock_iter = locks
-            .writable
-            .iter()
-            .map(|address| solana_scheduler::LockAttempt::new(self.preloader.load(**address), solana_scheduler::RequestedUsage::Writable));
-        let readonly_lock_iter = locks
-            .readonly
-            .iter()
-            .map(|address| solana_scheduler::LockAttempt::new(self.preloader.load(**address), solana_scheduler::RequestedUsage::Readonly));
-        let locks = writable_lock_iter.chain(readonly_lock_iter).collect::<Vec<_>>();
+        let writable_lock_iter = locks.writable.iter().map(|address| {
+            solana_scheduler::LockAttempt::new(
+                self.preloader.load(**address),
+                solana_scheduler::RequestedUsage::Writable,
+            )
+        });
+        let readonly_lock_iter = locks.readonly.iter().map(|address| {
+            solana_scheduler::LockAttempt::new(
+                self.preloader.load(**address),
+                solana_scheduler::RequestedUsage::Readonly,
+            )
+        });
+        let locks = writable_lock_iter
+            .chain(readonly_lock_iter)
+            .collect::<Vec<_>>();
 
         //assert_eq!(index, self.transaction_index.fetch_add(1, std::sync::atomic::Ordering::SeqCst));
         let uw = usize::max_value() - index;
-        let t = solana_scheduler::Task::new_for_queue(nast, uw as u64, (sanitized_tx.clone(), locks));
-        self.transaction_sender.as_ref().unwrap().send(solana_scheduler::SchedulablePayload(solana_scheduler::Flushable::Payload(t))).unwrap();
+        let t =
+            solana_scheduler::Task::new_for_queue(nast, uw as u64, (sanitized_tx.clone(), locks));
+        self.transaction_sender
+            .as_ref()
+            .unwrap()
+            .send(solana_scheduler::SchedulablePayload(
+                solana_scheduler::Flushable::Payload(t),
+            ))
+            .unwrap();
     }
 }
 
@@ -1380,89 +1408,109 @@ impl Scheduler {
         let collected_errors = Arc::new(std::sync::Mutex::new(Vec::new()));
         let collected_errors_in_collector_thread = Arc::clone(&collected_errors);
 
-        let error_collector_thread_handle = std::thread::Builder::new().name(format!("solScErrCol{:02}", 0)).spawn(move || {
-            let started = (cpu_time::ThreadTime::now(), std::time::Instant::now());
-            if max_thread_priority {
-                thread_priority::set_current_thread_priority(thread_priority::ThreadPriority::Max).unwrap();
-            }
-
-            while let Ok(solana_scheduler::ExaminablePayload(mut ee)) = retired_ee_receiver.recv() {
-                if send_metrics {
-                    let sig = ee.task.tx.0.signature().to_string();
+        let error_collector_thread_handle = std::thread::Builder::new()
+            .name(format!("solScErrCol{:02}", 0))
+            .spawn(move || {
+                let started = (cpu_time::ThreadTime::now(), std::time::Instant::now());
+                if max_thread_priority {
+                    thread_priority::set_current_thread_priority(
+                        thread_priority::ThreadPriority::Max,
+                    )
+                    .unwrap();
+                }
 
+                while let Ok(solana_scheduler::ExaminablePayload(mut ee)) =
+                    retired_ee_receiver.recv()
+                {
+                    if send_metrics {
+                        let sig = ee.task.tx.0.signature().to_string();
+
+                        datapoint_info_at!(
+                            ee.finish_time.unwrap(),
+                            "individual_tx_stats",
+                            ("slot", ee.slot, i64),
+                            ("index", ee.transaction_index, i64),
+                            ("thread", format!("solScExLane{:02}", ee.thx), String),
+                            ("signature", &sig, String),
+                            ("account_locks_in_json", "{}", String),
+                            (
+                                "status",
+                                format!("{:?}", ee.execution_result.as_ref().unwrap()),
+                                String
+                            ),
+                            ("duration", ee.execution_us, i64),
+                            ("cpu_duration", ee.execution_cpu_us, i64),
+                            ("compute_units", ee.cu, i64),
+                        );
+                    }
 
-                    datapoint_info_at!(
-                        ee.finish_time.unwrap(),
-                        "individual_tx_stats",
-                        ("slot", ee.slot, i64),
-                        ("index", ee.transaction_index, i64),
-                        ("thread", format!("solScExLane{:02}", ee.thx), String),
-                        ("signature", &sig, String),
-                        ("account_locks_in_json", "{}", String),
-                        ("status", format!("{:?}", ee.execution_result.as_ref().unwrap()), String),
-                        ("duration", ee.execution_us, i64),
-                        ("cpu_duration", ee.execution_cpu_us, i64),
-                        ("compute_units", ee.cu, i64),
-                    );
+                    if ee.is_aborted() {
+                        warn!(
+                            "scheduler: Unexpected validator error: {:?}, transaction: {:?}",
+                            ee.execution_result, ee.task.tx.0
+                        );
+                        collected_errors_in_collector_thread
+                            .lock()
+                            .unwrap()
+                            .push(ee.execution_result.take().unwrap());
+                    }
+                    drop(ee);
                 }
+                todo!();
 
-                        if ee.is_aborted() {
-                            warn!(
-                                "scheduler: Unexpected validator error: {:?}, transaction: {:?}",
-                                ee.execution_result, ee.task.tx.0
-                            );
-                            collected_errors_in_collector_thread.lock().unwrap().push(ee.execution_result.take().unwrap());
-                        }
-                        drop(ee);
-            }
-            todo!();
-
-            Ok((started.0.elapsed(), started.1.elapsed()))
-        }).unwrap();
+                Ok((started.0.elapsed(), started.1.elapsed()))
+            })
+            .unwrap();
 
         let random_id = rand::thread_rng().gen::<u64>();
 
-        let scheduler_thread_handle = std::thread::Builder::new().name("solScheduler".to_string()).spawn(move || {
-            let started = (cpu_time::ThreadTime::now(), std::time::Instant::now());
-            if max_thread_priority {
-                thread_priority::set_current_thread_priority(thread_priority::ThreadPriority::Max).unwrap();
-            }
+        let scheduler_thread_handle = std::thread::Builder::new()
+            .name("solScheduler".to_string())
+            .spawn(move || {
+                let started = (cpu_time::ThreadTime::now(), std::time::Instant::now());
+                if max_thread_priority {
+                    thread_priority::set_current_thread_priority(
+                        thread_priority::ThreadPriority::Max,
+                    )
+                    .unwrap();
+                }
 
-            let max_executing_queue_count = std::env::var("MAX_EXECUTING_QUEUE_COUNT")
-                .unwrap_or(format!("{}", 1))
-                .parse::<usize>()
-                .unwrap();
+                let max_executing_queue_count = std::env::var("MAX_EXECUTING_QUEUE_COUNT")
+                    .unwrap_or(format!("{}", 1))
+                    .parse::<usize>()
+                    .unwrap();
 
-            loop {
-                let mut runnable_queue = solana_scheduler::TaskQueue::default();
-                let maybe_checkpoint = solana_scheduler::ScheduleStage::run(
-                    random_id,
-                    max_executing_queue_count,
-                    &mut runnable_queue,
-                    &mut address_book,
-                    &transaction_receiver,
-                    &scheduled_ee_sender,
-                    Some(&scheduled_high_ee_sender),
-                    &processed_ee_receiver,
-                    Some(&retired_ee_sender),
-                );
+                loop {
+                    let mut runnable_queue = solana_scheduler::TaskQueue::default();
+                    let maybe_checkpoint = solana_scheduler::ScheduleStage::run(
+                        random_id,
+                        max_executing_queue_count,
+                        &mut runnable_queue,
+                        &mut address_book,
+                        &transaction_receiver,
+                        &scheduled_ee_sender,
+                        Some(&scheduled_high_ee_sender),
+                        &processed_ee_receiver,
+                        Some(&retired_ee_sender),
+                    );
 
-                if let Some(checkpoint) = maybe_checkpoint {
-                    checkpoint.wait_for_restart();
-                    continue;
-                } else {
-                    break;
+                    if let Some(checkpoint) = maybe_checkpoint {
+                        checkpoint.wait_for_restart();
+                        continue;
+                    } else {
+                        break;
+                    }
                 }
-            }
 
-            drop(transaction_receiver);
-            drop(scheduled_ee_sender);
-            drop(scheduled_high_ee_sender);
-            drop(processed_ee_receiver);
+                drop(transaction_receiver);
+                drop(scheduled_ee_sender);
+                drop(scheduled_high_ee_sender);
+                drop(processed_ee_receiver);
 
-            todo!();
-            Ok((started.0.elapsed(), started.1.elapsed()))
-        }).unwrap();
+                todo!();
+                Ok((started.0.elapsed(), started.1.elapsed()))
+            })
+            .unwrap();
 
         let s = Self {
             random_id,
@@ -1476,7 +1524,11 @@ impl Scheduler {
             bank,
             slot: Default::default(),
         };
-        info!("scheduler: id_{:016x} setup done with {}us", random_id, start.elapsed().as_micros());
+        info!(
+            "scheduler: id_{:016x} setup done with {}us",
+            random_id,
+            start.elapsed().as_micros()
+        );
 
         s
     }
@@ -1484,18 +1536,34 @@ impl Scheduler {
 
 impl Scheduler {
     fn gracefully_stop(&self) -> Result<()> {
-        if self.graceful_stop_initiated.load(std::sync::atomic::Ordering::SeqCst) {
-            info!("Scheduler::gracefully_stop(): id_{:016x} (skipped..?)", self.random_id);
+        if self
+            .graceful_stop_initiated
+            .load(std::sync::atomic::Ordering::SeqCst)
+        {
+            info!(
+                "Scheduler::gracefully_stop(): id_{:016x} (skipped..?)",
+                self.random_id
+            );
             return Ok(());
         }
-        self.graceful_stop_initiated.store(true, std::sync::atomic::Ordering::SeqCst);
+        self.graceful_stop_initiated
+            .store(true, std::sync::atomic::Ordering::SeqCst);
 
-        info!("Scheduler::gracefully_stop(): id_{:016x} waiting..", self.random_id);
+        info!(
+            "Scheduler::gracefully_stop(): id_{:016x} waiting..",
+            self.random_id
+        );
         //let transaction_sender = self.transaction_sender.take().unwrap();
 
         //drop(transaction_sender);
         let checkpoint = solana_scheduler::Checkpoint::new(2);
-        self.transaction_sender.as_ref().unwrap().send(solana_scheduler::SchedulablePayload(solana_scheduler::Flushable::Flush(std::sync::Arc::clone(&checkpoint)))).unwrap();
+        self.transaction_sender
+            .as_ref()
+            .unwrap()
+            .send(solana_scheduler::SchedulablePayload(
+                solana_scheduler::Flushable::Flush(std::sync::Arc::clone(&checkpoint)),
+            ))
+            .unwrap();
         checkpoint.wait_for_restart();
         {
             *self.bank.write().unwrap() = None;
@@ -4212,7 +4280,10 @@ impl Bank {
     /// reaches its max tick height. Can be called by tests to get new blockhashes for transaction
     /// processing without advancing to a new bank slot.
     pub fn register_recent_blockhash(&self, blockhash: &Hash) {
-        info!("register_recent_blockhash: slot: {} reinitializing the scheduler: start", self.slot());
+        info!(
+            "register_recent_blockhash: slot: {} reinitializing the scheduler: start",
+            self.slot()
+        );
 
         let maybe_last_error = self.wait_for_scheduler(false);
         let scheduler = SCHEDULER_POOL.lock().unwrap().take_from_pool();
@@ -4225,13 +4296,24 @@ impl Bank {
         let mut w_blockhash_queue = self.blockhash_queue.write().unwrap();
         //let new_scheduler = Scheduler::default();
         if maybe_last_error.is_err() {
-            warn!("register_recent_blockhash: carrying over this error: {:?}", maybe_last_error);
+            warn!(
+                "register_recent_blockhash: carrying over this error: {:?}",
+                maybe_last_error
+            );
             //new_scheduler.collected_errors.lock().unwrap().push(maybe_last_error);
-            s2.as_ref().unwrap().collected_errors.lock().unwrap().push(maybe_last_error);
+            s2.as_ref()
+                .unwrap()
+                .collected_errors
+                .lock()
+                .unwrap()
+                .push(maybe_last_error);
         }
         //*self.scheduler.write().unwrap() = new_scheduler;
 
-        info!("register_recent_blockhash: slot: {} reinitializing the scheduler: end", self.slot());
+        info!(
+            "register_recent_blockhash: slot: {} reinitializing the scheduler: end",
+            self.slot()
+        );
 
         w_blockhash_queue.register_hash(blockhash, self.fee_rate_governor.lamports_per_signature);
         self.update_recent_blockhashes_locked(&w_blockhash_queue);
@@ -4347,7 +4429,8 @@ impl Bank {
         txs: &'b [SanitizedTransaction],
     ) -> TransactionBatch<'a, 'b> {
         let tx_account_lock_limit = self.get_transaction_account_lock_limit();
-        let lock_results: Vec<Result<_>> = txs.iter()
+        let lock_results: Vec<Result<_>> = txs
+            .iter()
             .map(|tx| tx.get_account_locks(tx_account_lock_limit).map(|_| ()))
             .collect();
         let mut batch = TransactionBatch::new(lock_results, self, Cow::Borrowed(txs));
@@ -6589,7 +6672,10 @@ impl Bank {
         transaction_indexes: impl Iterator<Item = usize>,
     ) {
         assert_eq!(this_arced_bank.slot(), self.slot());
-        trace!("schedule_and_commit_transactions(): {} txs", transactions.len());
+        trace!(
+            "schedule_and_commit_transactions(): {} txs",
+            transactions.len()
+        );
 
         let s = {
             let r = self.scheduler2.read().unwrap();
@@ -6602,7 +6688,8 @@ impl Bank {
                 let ss = self.scheduler2.write().unwrap();
                 let w = ss.as_ref().unwrap();
                 *w.bank.write().unwrap() = Some(Arc::downgrade(&this_arced_bank));
-                w.slot.store(this_arced_bank.slot(), std::sync::atomic::Ordering::SeqCst);
+                w.slot
+                    .store(this_arced_bank.slot(), std::sync::atomic::Ordering::SeqCst);
                 drop(w);
                 drop(ss);
 
@@ -6610,7 +6697,13 @@ impl Bank {
                 info!("reconfigured scheduler to the bank slot: {}", self.slot());
                 s
             } else {
-                assert_eq!(this_arced_bank.slot(), r.as_ref().unwrap().slot.load(std::sync::atomic::Ordering::SeqCst));
+                assert_eq!(
+                    this_arced_bank.slot(),
+                    r.as_ref()
+                        .unwrap()
+                        .slot
+                        .load(std::sync::atomic::Ordering::SeqCst)
+                );
                 r
             }
         };
@@ -6621,9 +6714,7 @@ impl Bank {
         }
     }
 
-    pub fn handle_aborted_transactions(
-        &self,
-    ) -> Vec<Result<()>> {
+    pub fn handle_aborted_transactions(&self) -> Vec<Result<()>> {
         let s = self.scheduler2.read().unwrap();
         let scheduler = s.as_ref().unwrap();
         scheduler.handle_aborted_executions()
@@ -8284,12 +8375,19 @@ impl Bank {
 
         if let Some(scheduler) = s {
             scheduler.gracefully_stop().unwrap();
-            let e = scheduler.handle_aborted_executions().into_iter().next().unwrap_or(Ok(()));
+            let e = scheduler
+                .handle_aborted_executions()
+                .into_iter()
+                .next()
+                .unwrap_or(Ok(()));
             SCHEDULER_POOL.lock().unwrap().return_to_pool(scheduler);
             e
         } else {
             let current_thread_name = std::thread::current().name().unwrap().to_string();
-            warn!("Bank::wait_for_scheduler(via_drop: {}) skipped by {} ...", via_drop, current_thread_name);
+            warn!(
+                "Bank::wait_for_scheduler(via_drop: {}) skipped by {} ...",
+                via_drop, current_thread_name
+            );
 
             Ok(())
         }
@@ -8423,9 +8521,16 @@ impl Drop for Bank {
         if self.scheduler2.read().unwrap().is_some() {
             let r = self.wait_for_scheduler(true);
             if let Err(err) = r {
-                warn!("Bank::drop(): slot: {} discarding error from scheduler: {:?}", self.slot(), err);
+                warn!(
+                    "Bank::drop(): slot: {} discarding error from scheduler: {:?}",
+                    self.slot(),
+                    err
+                );
             } else {
-                trace!("Bank::drop(): slot: {} scheduler is returned to the pool", self.slot());
+                trace!(
+                    "Bank::drop(): slot: {} scheduler is returned to the pool",
+                    self.slot()
+                );
             }
         }
 
diff --git a/runtime/src/lib.rs b/runtime/src/lib.rs
index 18c4a0aabb33c0..e98bcadff687f7 100644
--- a/runtime/src/lib.rs
+++ b/runtime/src/lib.rs
@@ -1,5 +1,4 @@
 #![feature(negative_impls)]
-
 #![cfg_attr(RUSTC_WITH_SPECIALIZATION, feature(min_specialization))]
 #![allow(clippy::integer_arithmetic)]
 
diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 308930bd708dbe..e479414b3b6c4c 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -19,7 +19,11 @@ type PageRcInner<T> = std::rc::Rc<T>;
 unsafe impl Send for PageRc {}
 */
 
-type PageRcInner = triomphe::Arc<(std::cell::RefCell<Page>, TaskIds, std::sync::atomic::AtomicUsize)>;
+type PageRcInner = triomphe::Arc<(
+    std::cell::RefCell<Page>,
+    TaskIds,
+    std::sync::atomic::AtomicUsize,
+)>;
 
 #[derive(Debug, Clone)]
 pub struct PageRc(PageRcInner);
@@ -109,7 +113,9 @@ impl ExecutionEnvironment {
                 });
 
             if should_remove && lock_attempt.requested_usage == RequestedUsage::Writable {
-                lock_attempt.target_contended_write_task_count().fetch_sub(1, std::sync::atomic::Ordering::SeqCst);
+                lock_attempt
+                    .target_contended_write_task_count()
+                    .fetch_sub(1, std::sync::atomic::Ordering::SeqCst);
             }
         }
     }
@@ -132,7 +138,7 @@ pub unsafe trait NotAtScheduleThread: Copy {}
 
 impl PageRc {
     fn page_mut<AST: AtScheduleThread>(&self, _ast: AST) -> std::cell::RefMut<'_, Page> {
-        self.0.0.borrow_mut()
+        self.0 .0.borrow_mut()
     }
 }
 
@@ -175,11 +181,11 @@ impl LockAttempt {
     }
 
     pub fn target_contended_unique_weights(&self) -> &TaskIds {
-        &self.target.0.1
+        &self.target.0 .1
     }
 
     pub fn target_contended_write_task_count(&self) -> &std::sync::atomic::AtomicUsize {
-        &self.target.0.2
+        &self.target.0 .2
     }
 }
 
@@ -333,13 +339,22 @@ impl AddressBook {
         unique_weight: &UniqueWeight,
         attempt: &mut LockAttempt,
     ) -> CU {
-        let tcuw = attempt.target_contended_unique_weights().task_ids.back().map(|j| *j.key());
+        let tcuw = attempt
+            .target_contended_unique_weights()
+            .task_ids
+            .back()
+            .map(|j| *j.key());
 
         let strictly_lockable_for_replay = if tcuw.is_none() {
             true
         } else if tcuw.unwrap() == *unique_weight {
             true
-        } else if attempt.requested_usage == RequestedUsage::Readonly && attempt.target_contended_write_task_count().load(std::sync::atomic::Ordering::SeqCst) == 0 {
+        } else if attempt.requested_usage == RequestedUsage::Readonly
+            && attempt
+                .target_contended_write_task_count()
+                .load(std::sync::atomic::Ordering::SeqCst)
+                == 0
+        {
             true
         } else {
             false
@@ -351,7 +366,6 @@ impl AddressBook {
             return page.cu;
         }
 
-
         let LockAttempt {
             target,
             requested_usage,
@@ -360,7 +374,6 @@ impl AddressBook {
         } = attempt;
         let mut page = target.page_mut(ast);
 
-
         let next_usage = page.next_usage;
         match page.current_usage {
             Usage::Unused => {
@@ -611,14 +624,14 @@ impl Task {
         &self,
         _ast: AST,
     ) -> std::cell::RefMut<'_, Vec<LockAttempt>> {
-        self.tx.1.0.borrow_mut()
+        self.tx.1 .0.borrow_mut()
     }
 
     fn lock_attempts_not_mut<NAST: NotAtScheduleThread>(
         &self,
         _nast: NAST,
     ) -> std::cell::Ref<'_, Vec<LockAttempt>> {
-        self.tx.1.0.borrow()
+        self.tx.1 .0.borrow()
     }
 
     fn update_busiest_page_cu(&self, cu: CU) {
@@ -655,7 +668,8 @@ impl Task {
 
     pub fn record_execute_time(&self, queue_clock: usize, execute_clock: usize) {
         //self.queue_end_time.store(queue_clock, std::sync::atomic::Ordering::SeqCst);
-        self.execute_time.store(execute_clock, std::sync::atomic::Ordering::SeqCst);
+        self.execute_time
+            .store(execute_clock, std::sync::atomic::Ordering::SeqCst);
     }
 
     pub fn execute_time(&self) -> usize {
@@ -752,10 +766,14 @@ impl Task {
         task_sender: &crossbeam_channel::Sender<(TaskInQueue, Vec<LockAttempt>)>,
     ) {
         for lock_attempt in &*this.lock_attempts_mut(ast) {
-            lock_attempt.target_contended_unique_weights().insert_task(this.unique_weight, Task::clone_in_queue(this));
+            lock_attempt
+                .target_contended_unique_weights()
+                .insert_task(this.unique_weight, Task::clone_in_queue(this));
 
             if lock_attempt.requested_usage == RequestedUsage::Writable {
-                lock_attempt.target_contended_write_task_count().fetch_add(1, std::sync::atomic::Ordering::SeqCst);
+                lock_attempt
+                    .target_contended_write_task_count()
+                    .fetch_add(1, std::sync::atomic::Ordering::SeqCst);
             }
         }
         //let a = Task::clone_in_queue(this);
@@ -849,7 +867,11 @@ fn attempt_lock_for_execution<'a, AST: AtScheduleThread>(
         match attempt.status {
             LockStatus::Succeded => {}
             LockStatus::Failed => {
-                trace!("lock failed: {}/{:?}", attempt.target.page_mut(ast).address_str, attempt.requested_usage);
+                trace!(
+                    "lock failed: {}/{:?}",
+                    attempt.target.page_mut(ast).address_str,
+                    attempt.requested_usage
+                );
                 unlockable_count += 1;
             }
             LockStatus::Provisional => {
@@ -891,7 +913,6 @@ enum TaskSelection {
     OnlyFromContended(usize),
 }
 
-
 impl TaskSelection {
     fn should_proceed(&self) -> bool {
         match self {
@@ -954,7 +975,10 @@ impl ScheduleStage {
                 let uw = weight_from_contended.key();
 
                 if weight_from_runnable > uw {
-                    panic!("replay shouldn't see this branch: {} > {}", weight_from_runnable, uw);
+                    panic!(
+                        "replay shouldn't see this branch: {} > {}",
+                        weight_from_runnable, uw
+                    );
 
                     /*
                     trace!("select: runnable > contended");
@@ -1027,9 +1051,12 @@ impl ScheduleStage {
         }
 
         loop {
-            if let Some((task_source, next_task)) =
-                Self::select_next_task(runnable_queue, address_book, contended_count, task_selection)
-            {
+            if let Some((task_source, next_task)) = Self::select_next_task(
+                runnable_queue,
+                address_book,
+                contended_count,
+                task_selection,
+            ) {
                 let from_runnable = task_source == TaskSource::Runnable;
                 if from_runnable {
                     next_task.record_queue_time(*sequence_clock, *queue_clock);
@@ -1137,7 +1164,7 @@ impl ScheduleStage {
                         */
 
                         //address_book.uncontended_task_ids.insert(next_task.unique_weight, next_task);
-                        
+
                         break;
                     } else {
                         unreachable!();
@@ -1424,7 +1451,10 @@ impl ScheduleStage {
         sequence_time: &mut usize,
     ) {
         weighted_tx.record_sequence_time(*sequence_time);
-        assert_eq!(*sequence_time, weighted_tx.transaction_index_in_entries_for_replay() as usize);
+        assert_eq!(
+            *sequence_time,
+            weighted_tx.transaction_index_in_entries_for_replay() as usize
+        );
         *sequence_time = sequence_time.checked_add(1).unwrap();
         Self::push_to_runnable_queue(weighted_tx, runnable_queue)
     }
@@ -1462,63 +1492,70 @@ impl ScheduleStage {
 
         let (ee_sender, ee_receiver) = crossbeam_channel::unbounded::<ExaminablePayload>();
 
-        let (to_next_stage, maybe_reaper_thread_handle) = if let Some(to_next_stage) = maybe_to_next_stage {
-            (to_next_stage, None)
-        } else {
-            todo!();
-            let h = std::thread::Builder::new()
-                .name("solScReaper".to_string())
-                .spawn(move || {
-                    #[derive(Clone, Copy, Debug)]
-                    struct NotAtTopOfScheduleThread;
-                    unsafe impl NotAtScheduleThread for NotAtTopOfScheduleThread {}
-                    let nast = NotAtTopOfScheduleThread;
-
-                    while let Ok(ExaminablePayload(a)) = ee_receiver.recv() {
-                                assert!(a.task.lock_attempts_not_mut(nast).is_empty());
-                                //assert!(a.task.sequence_time() != usize::max_value());
-                                //let lock_attempts = std::mem::take(&mut a.lock_attempts);
-                                //drop(lock_attempts);
-                                //TaskInQueue::get_mut(&mut a.task).unwrap();
-                    }
-                    assert_eq!(ee_receiver.len(), 0);
-                    Ok::<(), ()>(())
-                })
-                .unwrap();
+        let (to_next_stage, maybe_reaper_thread_handle) =
+            if let Some(to_next_stage) = maybe_to_next_stage {
+                (to_next_stage, None)
+            } else {
+                todo!();
+                let h = std::thread::Builder::new()
+                    .name("solScReaper".to_string())
+                    .spawn(move || {
+                        #[derive(Clone, Copy, Debug)]
+                        struct NotAtTopOfScheduleThread;
+                        unsafe impl NotAtScheduleThread for NotAtTopOfScheduleThread {}
+                        let nast = NotAtTopOfScheduleThread;
+
+                        while let Ok(ExaminablePayload(a)) = ee_receiver.recv() {
+                            assert!(a.task.lock_attempts_not_mut(nast).is_empty());
+                            //assert!(a.task.sequence_time() != usize::max_value());
+                            //let lock_attempts = std::mem::take(&mut a.lock_attempts);
+                            //drop(lock_attempts);
+                            //TaskInQueue::get_mut(&mut a.task).unwrap();
+                        }
+                        assert_eq!(ee_receiver.len(), 0);
+                        Ok::<(), ()>(())
+                    })
+                    .unwrap();
 
-            (&ee_sender, Some(h))
-        };
+                (&ee_sender, Some(h))
+            };
         let (task_sender, task_receiver) =
             crossbeam_channel::unbounded::<(TaskInQueue, Vec<LockAttempt>)>();
         let indexer_count = std::env::var("INDEXER_COUNT")
             .unwrap_or(format!("{}", 4))
             .parse::<usize>()
             .unwrap();
-        let indexer_handles = (0..indexer_count).map(|thx| {
-            let task_receiver = task_receiver.clone();
-            std::thread::Builder::new()
-                .name(format!("solScIdxer{:02}", thx))
-                .spawn(move || {
-                    while let Ok((task, ll)) = task_receiver.recv() {
-                        for lock_attempt in ll {
-                            if task.already_finished() {
-                                break;
+        let indexer_handles = (0..indexer_count)
+            .map(|thx| {
+                let task_receiver = task_receiver.clone();
+                std::thread::Builder::new()
+                    .name(format!("solScIdxer{:02}", thx))
+                    .spawn(move || {
+                        while let Ok((task, ll)) = task_receiver.recv() {
+                            for lock_attempt in ll {
+                                if task.already_finished() {
+                                    break;
+                                }
+                                lock_attempt
+                                    .target_contended_unique_weights()
+                                    .insert_task(task.unique_weight, Task::clone_in_queue(&task));
+                                todo!("contended_write_task_count!");
                             }
-                            lock_attempt
-                                .target_contended_unique_weights()
-                                .insert_task(task.unique_weight, Task::clone_in_queue(&task));
-                            todo!("contended_write_task_count!");
                         }
-                    }
-                    assert_eq!(task_receiver.len(), 0);
-                    Ok::<(), ()>(())
-                })
-                .unwrap()
-        }).collect::<Vec<_>>();
+                        assert_eq!(task_receiver.len(), 0);
+                        Ok::<(), ()>(())
+                    })
+                    .unwrap()
+            })
+            .collect::<Vec<_>>();
 
         let no_aggresive_contended = std::env::var("NO_AGGRESSIVE_CONTENDED").is_ok();
 
-        let (mut from_disconnected, mut from_exec_disconnected, mut no_more_work): (bool, bool, bool) = Default::default();
+        let (mut from_disconnected, mut from_exec_disconnected, mut no_more_work): (
+            bool,
+            bool,
+            bool,
+        ) = Default::default();
 
         let mut maybe_checkpoint = None;
 
@@ -1577,10 +1614,15 @@ impl ScheduleStage {
                 select_skipped = true;
             }
 
-           no_more_work = from_disconnected && runnable_queue.task_count() + contended_count + executing_queue_count + provisioning_tracker_count == 0;
-           if from_disconnected && (from_exec_disconnected || no_more_work) {
-               break;
-           }
+            no_more_work = from_disconnected
+                && runnable_queue.task_count()
+                    + contended_count
+                    + executing_queue_count
+                    + provisioning_tracker_count
+                    == 0;
+            if from_disconnected && (from_exec_disconnected || no_more_work) {
+                break;
+            }
 
             let mut first_iteration = true;
             let (mut empty_from, mut empty_from_exec) = (false, false);
@@ -1589,7 +1631,11 @@ impl ScheduleStage {
             loop {
                 let runnable_finished = from_disconnected && runnable_queue.has_no_task();
 
-                let mut selection = TaskSelection::OnlyFromContended(if runnable_finished { usize::max_value() } else { usize::max_value() /*2*/ });
+                let mut selection = TaskSelection::OnlyFromContended(if runnable_finished {
+                    usize::max_value()
+                } else {
+                    usize::max_value() /*2*/
+                });
                 while !address_book.uncontended_task_ids.is_empty() && selection.should_proceed() {
                     let prefer_immediate = true; //provisioning_tracker_count / 4 > executing_queue_count;
 
@@ -1609,7 +1655,10 @@ impl ScheduleStage {
                     );
                     if let Some(ee) = maybe_ee {
                         executing_queue_count = executing_queue_count.checked_add(1).unwrap();
-                        to_high_execute_substage.unwrap_or(to_execute_substage).send(ExecutablePayload(ee)).unwrap();
+                        to_high_execute_substage
+                            .unwrap_or(to_execute_substage)
+                            .send(ExecutablePayload(ee))
+                            .unwrap();
                     }
                     debug!("schedule_once id_{:016x} [C] ch(prev: {}, exec: {}+{}|{}), r: {}, u/c: {}/{}, (imm+provi)/max: ({}+{})/{} s: {} l(s+f): {}+{}", random_id, (if from_disconnected { "-".to_string() } else { format!("{}", from_prev.len()) }), to_high_execute_substage.map(|t| format!("{}", t.len())).unwrap_or("-".into()), to_execute_substage.len(), from_exec.len(), runnable_queue.task_count(), address_book.uncontended_task_ids.len(), contended_count, executing_queue_count, provisioning_tracker_count, max_executing_queue_count, address_book.stuck_tasks.len(), processed_count, failed_lock_count);
 
@@ -1625,12 +1674,18 @@ impl ScheduleStage {
                             let delta = (processed_count - last_processed_count) as u128;
                             let elapsed2 = elapsed.as_micros();
                             info!("schedule_once:interval id_{:016x} ch(prev: {}, exec: {}+{}|{}), r: {}, u/c: {}/{}, (imm+provi)/max: ({}+{})/{} s: {} l(s+f): {}+{} ({}txs/{}us={}tps)", random_id, (if from_disconnected { "-".to_string() } else { format!("{}", from_prev.len()) }), to_high_execute_substage.map(|t| format!("{}", t.len())).unwrap_or("-".into()), to_execute_substage.len(), from_exec.len(), runnable_queue.task_count(), address_book.uncontended_task_ids.len(), contended_count, executing_queue_count, provisioning_tracker_count, max_executing_queue_count, address_book.stuck_tasks.len(), processed_count, failed_lock_count, delta, elapsed.as_micros(), 1_000_000_u128*delta/elapsed2);
-                            (last_time, last_processed_count) = (Some(std::time::Instant::now()), processed_count);
+                            (last_time, last_processed_count) =
+                                (Some(std::time::Instant::now()), processed_count);
                         }
                     }
                 }
                 let mut selection = TaskSelection::OnlyFromRunnable;
-                while !runnable_queue.has_no_task() && selection.should_proceed() && (to_high_execute_substage.is_some() || executing_queue_count + provisioning_tracker_count < max_executing_queue_count) {
+                while !runnable_queue.has_no_task()
+                    && selection.should_proceed()
+                    && (to_high_execute_substage.is_some()
+                        || executing_queue_count + provisioning_tracker_count
+                            < max_executing_queue_count)
+                {
                     let prefer_immediate = true; //provisioning_tracker_count / 4 > executing_queue_count;
 
                     let maybe_ee = Self::schedule_next_execution(
@@ -1654,7 +1709,9 @@ impl ScheduleStage {
                     debug!("schedule_once id_{:016x} [R] ch(prev: {}, exec: {}+{}|{}), r: {}, u/c: {}/{}, (imm+provi)/max: ({}+{})/{} s: {} l(s+f): {}+{}", random_id, (if from_disconnected { "-".to_string() } else { format!("{}", from_prev.len()) }), to_high_execute_substage.map(|t| format!("{}", t.len())).unwrap_or("-".into()), to_execute_substage.len(), from_exec.len(), runnable_queue.task_count(), address_book.uncontended_task_ids.len(), contended_count, executing_queue_count, provisioning_tracker_count, max_executing_queue_count, address_book.stuck_tasks.len(), processed_count, failed_lock_count);
 
                     if !from_exec.is_empty() {
-                        trace!("abort aggressive runnable queue processing due to non-empty from_exec");
+                        trace!(
+                            "abort aggressive runnable queue processing due to non-empty from_exec"
+                        );
                         break;
                     }
 
@@ -1665,7 +1722,8 @@ impl ScheduleStage {
                             let delta = (processed_count - last_processed_count) as u128;
                             let elapsed2 = elapsed.as_micros();
                             info!("schedule_once:interval id_{:016x} ch(prev: {}, exec: {}+{}|{}), r: {}, u/c: {}/{}, (imm+provi)/max: ({}+{})/{} s: {} l(s+f): {}+{} ({}txs/{}us={}tps)", random_id, (if from_disconnected { "-".to_string() } else { format!("{}", from_prev.len()) }), to_high_execute_substage.map(|t| format!("{}", t.len())).unwrap_or("-".into()), to_execute_substage.len(), from_exec.len(), runnable_queue.task_count(), address_book.uncontended_task_ids.len(), contended_count, executing_queue_count, provisioning_tracker_count, max_executing_queue_count, address_book.stuck_tasks.len(), processed_count, failed_lock_count, delta, elapsed.as_micros(), 1_000_000_u128*delta/elapsed2);
-                            (last_time, last_processed_count) = (Some(std::time::Instant::now()), processed_count);
+                            (last_time, last_processed_count) =
+                                (Some(std::time::Instant::now()), processed_count);
                         }
                     }
                 }
@@ -1701,7 +1759,9 @@ impl ScheduleStage {
                             &mut commit_clock,
                             &mut provisioning_tracker_count,
                         );
-                        to_next_stage.send(ExaminablePayload(processed_execution_environment)).unwrap();
+                        to_next_stage
+                            .send(ExaminablePayload(processed_execution_environment))
+                            .unwrap();
                     }
                     if !empty_from {
                         let SchedulablePayload(schedulable) = from_prev.recv().unwrap();
@@ -1709,17 +1769,21 @@ impl ScheduleStage {
                         empty_from = from_len == 0;
                         match schedulable {
                             Flushable::Flush(checkpoint) => {
-                               assert!(empty_from);
-                               assert_eq!(from_prev.len(), 0);
-                               assert!(!from_disconnected);
-                               from_disconnected = true;
-                               from_prev = never;
-                               assert!(maybe_checkpoint.is_none());
-                               maybe_checkpoint = Some(checkpoint);
-                            },
+                                assert!(empty_from);
+                                assert_eq!(from_prev.len(), 0);
+                                assert!(!from_disconnected);
+                                from_disconnected = true;
+                                from_prev = never;
+                                assert!(maybe_checkpoint.is_none());
+                                maybe_checkpoint = Some(checkpoint);
+                            }
                             Flushable::Payload(task) => {
-                                Self::register_runnable_task(task, runnable_queue, &mut sequence_time);
-                            },
+                                Self::register_runnable_task(
+                                    task,
+                                    runnable_queue,
+                                    &mut sequence_time,
+                                );
+                            }
                         }
                     }
                 }
@@ -1738,12 +1802,11 @@ impl ScheduleStage {
             indexer_handle.join().unwrap().unwrap();
         }
 
-
         if let Some(start_time) = maybe_start_time {
             let elapsed = start_time.elapsed();
             let elapsed2 = elapsed.as_micros();
             let tps_label = if elapsed2 > 0 {
-                format!("{}", 1_000_000_u128*(processed_count as u128)/elapsed2)
+                format!("{}", 1_000_000_u128 * (processed_count as u128) / elapsed2)
             } else {
                 "-".into()
             };
@@ -1796,27 +1859,48 @@ impl Checkpoint {
     pub fn wait_for_restart(&self) {
         let current_thread_name = std::thread::current().name().unwrap().to_string();
         let mut remaining_threads_guard = self.0.lock().unwrap();
-        info!("Checkpoint::wait_for_restart: {} is entering at {} -> {}", current_thread_name, *remaining_threads_guard, *remaining_threads_guard - 1);
+        info!(
+            "Checkpoint::wait_for_restart: {} is entering at {} -> {}",
+            current_thread_name,
+            *remaining_threads_guard,
+            *remaining_threads_guard - 1
+        );
 
         *remaining_threads_guard -= 1;
 
         if *remaining_threads_guard == 0 {
             drop(remaining_threads_guard);
             self.1.notify_all();
-            info!("Checkpoint::wait_for_restart: {} notified all others...", current_thread_name);
+            info!(
+                "Checkpoint::wait_for_restart: {} notified all others...",
+                current_thread_name
+            );
         } else {
-            info!("Checkpoint::wait_for_restart: {} is paused...", current_thread_name);
-            let _ = *self.1.wait_while(remaining_threads_guard, |&mut remaining_threads| remaining_threads > 0).unwrap();
-            info!("Checkpoint::wait_for_restart: {} is started...", current_thread_name);
+            info!(
+                "Checkpoint::wait_for_restart: {} is paused...",
+                current_thread_name
+            );
+            let _ = *self
+                .1
+                .wait_while(remaining_threads_guard, |&mut remaining_threads| {
+                    remaining_threads > 0
+                })
+                .unwrap();
+            info!(
+                "Checkpoint::wait_for_restart: {} is started...",
+                current_thread_name
+            );
         }
     }
 
     pub fn new(remaining_threads: usize) -> std::sync::Arc<Self> {
-        std::sync::Arc::new(Self(std::sync::Mutex::new(remaining_threads), std::sync::Condvar::new()))
+        std::sync::Arc::new(Self(
+            std::sync::Mutex::new(remaining_threads),
+            std::sync::Condvar::new(),
+        ))
     }
 }
 
-
 pub enum Flushable<T> {
     Payload(T),
     Flush(std::sync::Arc<Checkpoint>),
diff --git a/transaction-scheduler-bench/src/main.rs b/transaction-scheduler-bench/src/main.rs
index ee1dcb1439abaa..b729c0de1f6556 100644
--- a/transaction-scheduler-bench/src/main.rs
+++ b/transaction-scheduler-bench/src/main.rs
@@ -3,9 +3,7 @@ use {
     crossbeam_channel::{select, Receiver, Sender},
     log::*,
     rand::Rng,
-    solana_core::{
-        transaction_scheduler::TransactionScheduler,
-    },
+    solana_core::transaction_scheduler::TransactionScheduler,
     solana_measure::measure,
     solana_perf::packet::{Packet, PacketBatch},
     solana_runtime::{bank::Bank, bank_forks::BankForks},
@@ -117,33 +115,36 @@ struct PacketSendingConfig {
 }
 
 fn spawn_unified_scheduler(
-        mut address_book: solana_scheduler::AddressBook,
-        num_execution_threads: usize,
-        packet_batch_receiver: Receiver<solana_scheduler::SchedulablePayload>,
-        transaction_batch_senders: Vec<Sender<TransactionBatchMessage>>,
-        completed_transaction_receiver: Receiver<CompletedTransactionMessage>,
-        bank_forks: Arc<RwLock<BankForks>>,
-        max_batch_size: usize,
-        exit: Arc<AtomicBool>,
+    mut address_book: solana_scheduler::AddressBook,
+    num_execution_threads: usize,
+    packet_batch_receiver: Receiver<solana_scheduler::SchedulablePayload>,
+    transaction_batch_senders: Vec<Sender<TransactionBatchMessage>>,
+    completed_transaction_receiver: Receiver<CompletedTransactionMessage>,
+    bank_forks: Arc<RwLock<BankForks>>,
+    max_batch_size: usize,
+    exit: Arc<AtomicBool>,
 ) -> JoinHandle<()> {
     let lane_channel_factor = std::env::var("LANE_CHANNEL_FACTOR")
         .unwrap_or(format!("{}", 10))
         .parse::<usize>()
         .unwrap();
 
-    std::thread::Builder::new().name("sol-scheduler".to_string()).spawn(move || {
-        let mut runnable_queue = solana_scheduler::TaskQueue::default();
-
-        solana_scheduler::ScheduleStage::run(
-            num_execution_threads * lane_channel_factor,
-            &mut runnable_queue,
-            &mut address_book,
-            &packet_batch_receiver.clone(),
-            &transaction_batch_senders[0],
-            &completed_transaction_receiver,
-            None,//&completed_transaction_receiver
-        );
-    }).unwrap()
+    std::thread::Builder::new()
+        .name("sol-scheduler".to_string())
+        .spawn(move || {
+            let mut runnable_queue = solana_scheduler::TaskQueue::default();
+
+            solana_scheduler::ScheduleStage::run(
+                num_execution_threads * lane_channel_factor,
+                &mut runnable_queue,
+                &mut address_book,
+                &packet_batch_receiver.clone(),
+                &transaction_batch_senders[0],
+                &completed_transaction_receiver,
+                None, //&completed_transaction_receiver
+            );
+        })
+        .unwrap()
 }
 
 fn main() {
@@ -168,7 +169,8 @@ fn main() {
     assert!(high_conflict_sender <= num_accounts);
 
     let (packet_batch_sender, packet_batch_receiver) = crossbeam_channel::unbounded();
-    let (completed_transaction_sender, completed_transaction_receiver) = crossbeam_channel::unbounded();
+    let (completed_transaction_sender, completed_transaction_receiver) =
+        crossbeam_channel::unbounded();
     let (transaction_batch_senders, transaction_batch_receivers) =
         build_channels(num_execution_threads);
     let bank_forks = Arc::new(RwLock::new(BankForks::new(Bank::default_for_tests())));
@@ -222,28 +224,31 @@ fn main() {
     );
 
     // Spawn thread for reporting metrics
-    std::thread::Builder::new().name("sol-metrics".to_string()).spawn({
-        move || {
-            let start = Instant::now();
-            loop {
-                if exit.load(Ordering::Relaxed) {
-                    break;
-                }
-                if start.elapsed() > duration {
-                    let pending_transactions =
-                        metrics.num_transactions_sent.load(Ordering::Relaxed)
-                            - metrics.num_transactions_completed.load(Ordering::Relaxed);
-                    if pending_transactions == 0 {
+    std::thread::Builder::new()
+        .name("sol-metrics".to_string())
+        .spawn({
+            move || {
+                let start = Instant::now();
+                loop {
+                    if exit.load(Ordering::Relaxed) {
                         break;
                     }
-                }
+                    if start.elapsed() > duration {
+                        let pending_transactions =
+                            metrics.num_transactions_sent.load(Ordering::Relaxed)
+                                - metrics.num_transactions_completed.load(Ordering::Relaxed);
+                        if pending_transactions == 0 {
+                            break;
+                        }
+                    }
 
-                metrics.report();
-                std::thread::sleep(Duration::from_millis(100));
+                    metrics.report();
+                    std::thread::sleep(Duration::from_millis(100));
+                }
+                exit.store(true, Ordering::Relaxed);
             }
-            exit.store(true, Ordering::Relaxed);
-        }
-    }).unwrap();
+        })
+        .unwrap();
 
     scheduler_handle.join().unwrap();
     execution_handles
@@ -285,16 +290,19 @@ fn start_execution_thread(
     execution_per_tx_us: u64,
     exit: Arc<AtomicBool>,
 ) -> JoinHandle<()> {
-    std::thread::Builder::new().name(format!("sol-exec-{}", thread_index)).spawn(move || {
-        execution_worker(
-            metrics,
-            thread_index,
-            transaction_batch_receiver,
-            completed_transaction_sender,
-            execution_per_tx_us,
-            exit,
-        )
-    }).unwrap()
+    std::thread::Builder::new()
+        .name(format!("sol-exec-{}", thread_index))
+        .spawn(move || {
+            execution_worker(
+                metrics,
+                thread_index,
+                transaction_batch_receiver,
+                completed_transaction_sender,
+                execution_per_tx_us,
+                exit,
+            )
+        })
+        .unwrap()
 }
 
 fn execution_worker(
@@ -347,11 +355,18 @@ fn handle_transaction_batch(
     }
     */
     sleep(Duration::from_micros(
-        rand::thread_rng().gen_range(0..=execution_per_tx_us)
+        rand::thread_rng().gen_range(0..=execution_per_tx_us),
     ));
 
     use solana_runtime::transaction_priority_details::GetTransactionPriorityDetails;
-    let priority_collected = transaction_batch.0.task.tx.0.get_transaction_priority_details().unwrap().priority;
+    let priority_collected = transaction_batch
+        .0
+        .task
+        .tx
+        .0
+        .get_transaction_priority_details()
+        .unwrap()
+        .priority;
 
     metrics
         .num_transactions_completed
@@ -418,20 +433,23 @@ fn spawn_packet_sender(
     duration: Duration,
     exit: Arc<AtomicBool>,
 ) -> JoinHandle<()> {
-    std::thread::Builder::new().name(format!("sol-producer{:02}", i)).spawn(move || {
-        send_packets(
-            unique_weight,
-            producer_count,
-            preloader,
-            metrics,
-            num_accounts,
-            accounts,
-            packet_batch_sender,
-            config,
-            duration,
-            exit,
-        );
-    }).unwrap()
+    std::thread::Builder::new()
+        .name(format!("sol-producer{:02}", i))
+        .spawn(move || {
+            send_packets(
+                unique_weight,
+                producer_count,
+                preloader,
+                metrics,
+                num_accounts,
+                accounts,
+                packet_batch_sender,
+                config,
+                duration,
+                exit,
+            );
+        })
+        .unwrap()
 }
 
 pub fn get_transaction_priority_details(tx: &SanitizedTransaction) -> u64 {
@@ -500,10 +518,11 @@ fn send_packets(
                 let uw = unique_weight.fetch_sub(1, std::sync::atomic::Ordering::SeqCst);
                 let p = (p << 32) | (uw & 0x0000_0000_ffff_ffff);
                 let t = solana_scheduler::Task::new_for_queue(nast, p, v);
-                packet_batch_sender.send(solana_scheduler::SchedulablePayload(t)).unwrap();
+                packet_batch_sender
+                    .send(solana_scheduler::SchedulablePayload(t))
+                    .unwrap();
             }
         }
-        
 
         std::thread::sleep(loop_duration.saturating_sub(packet_build_time.as_duration()));
     }
@@ -555,26 +574,26 @@ fn build_packet(
             read_account_metas.chain(write_account_metas).collect(),
         ),
     ];
-    let transaction = Transaction::new_with_payer(
-        &ixs,
-        Some(&sending_keypair.pubkey()),
-    );
+    let transaction = Transaction::new_with_payer(&ixs, Some(&sending_keypair.pubkey()));
 
-    let sanitized_tx = SanitizedTransaction::try_from_legacy_transaction(
-        transaction,
-    )
-    .unwrap();
+    let sanitized_tx = SanitizedTransaction::try_from_legacy_transaction(transaction).unwrap();
 
     let locks = sanitized_tx.get_account_locks_unchecked();
-    let writable_lock_iter = locks
-        .writable
-        .iter()
-        .map(|address| solana_scheduler::LockAttempt::new(preloader.load(**address), solana_scheduler::RequestedUsage::Writable));
-    let readonly_lock_iter = locks
-        .readonly
-        .iter()
-        .map(|address| solana_scheduler::LockAttempt::new(preloader.load(**address), solana_scheduler::RequestedUsage::Readonly));
-    let locks = writable_lock_iter.chain(readonly_lock_iter).collect::<Vec<_>>();
+    let writable_lock_iter = locks.writable.iter().map(|address| {
+        solana_scheduler::LockAttempt::new(
+            preloader.load(**address),
+            solana_scheduler::RequestedUsage::Writable,
+        )
+    });
+    let readonly_lock_iter = locks.readonly.iter().map(|address| {
+        solana_scheduler::LockAttempt::new(
+            preloader.load(**address),
+            solana_scheduler::RequestedUsage::Readonly,
+        )
+    });
+    let locks = writable_lock_iter
+        .chain(readonly_lock_iter)
+        .collect::<Vec<_>>();
 
     (sanitized_tx, locks)
 }

From 62d5f85bd08b25509d207fbd70ed2f1d334574d4 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Wed, 21 Sep 2022 13:26:07 +0900
Subject: [PATCH 2892/3199] save

---
 scheduler/src/lib.rs | 17 ++++++++++++++---
 1 file changed, 14 insertions(+), 3 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index e479414b3b6c4c..0676e8437b74f7 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -839,6 +839,14 @@ impl TaskQueue {
     }
 }
 
+struct ChannelBackedTaskQueue {
+}
+
+impl ChannelBackedTaskQueue {
+    fn new(a: usize) -> Self {
+    }
+}
+
 #[inline(never)]
 fn attempt_lock_for_execution<'a, AST: AtScheduleThread>(
     ast: AST,
@@ -1558,6 +1566,7 @@ impl ScheduleStage {
         ) = Default::default();
 
         let mut maybe_checkpoint = None;
+        let mut channel_backed_runnable_queue = ChannelBackedTaskQueue::new(from_prev);
 
         loop {
             let mut select_skipped = false;
@@ -1587,7 +1596,8 @@ impl ScheduleStage {
                        }
                        match maybe_from {
                            Ok(SchedulablePayload(Flushable::Payload(task))) => {
-                                Self::register_runnable_task(task, runnable_queue, &mut sequence_time);
+                                //Self::register_runnable_task(task, runnable_queue, &mut sequence_time);
+                                channel_backed_runnable_queue.buffer(task);
                            },
                            Ok(SchedulablePayload(Flushable::Flush(checkpoint))) => {
                                assert_eq!(from_prev.len(), 0);
@@ -1778,11 +1788,12 @@ impl ScheduleStage {
                                 maybe_checkpoint = Some(checkpoint);
                             }
                             Flushable::Payload(task) => {
-                                Self::register_runnable_task(
+                                /*Self::register_runnable_task(
                                     task,
                                     runnable_queue,
                                     &mut sequence_time,
-                                );
+                                );*/
+                                channel_backed_runnable_queue.buffer(task);
                             }
                         }
                     }

From babb20c77de1c41e652fda7cbb6d3653511ef20e Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Wed, 21 Sep 2022 13:26:40 +0900
Subject: [PATCH 2893/3199] save

---
 scheduler/src/lib.rs | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 0676e8437b74f7..f73c55a5ee8130 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -839,10 +839,11 @@ impl TaskQueue {
     }
 }
 
-struct ChannelBackedTaskQueue {
+struct ChannelBackedTaskQueue<'a> {
+    channel: &'a crossbeam_channel::Receiver<SchedulablePayload>
 }
 
-impl ChannelBackedTaskQueue {
+impl<'a> ChannelBackedTaskQueue<'a> {
     fn new(a: usize) -> Self {
     }
 }

From dc37d8c29be42c012f5f185b286caa6f5119feba Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Wed, 21 Sep 2022 13:27:02 +0900
Subject: [PATCH 2894/3199] save

---
 scheduler/src/lib.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index f73c55a5ee8130..2c0136aa716065 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -840,11 +840,11 @@ impl TaskQueue {
 }
 
 struct ChannelBackedTaskQueue<'a> {
-    channel: &'a crossbeam_channel::Receiver<SchedulablePayload>
+    channel: &'a crossbeam_channel::Receiver<SchedulablePayload>,
 }
 
 impl<'a> ChannelBackedTaskQueue<'a> {
-    fn new(a: usize) -> Self {
+    fn new(a: &'a crossbeam_channel::Receiver<SchedulablePayload>) -> Self {
     }
 }
 

From 597bb14d8ff2f1ad4c748cdb82920b35f08305b3 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Wed, 21 Sep 2022 13:27:20 +0900
Subject: [PATCH 2895/3199] save

---
 scheduler/src/lib.rs | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 2c0136aa716065..5645fdb2541d8a 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -844,7 +844,8 @@ struct ChannelBackedTaskQueue<'a> {
 }
 
 impl<'a> ChannelBackedTaskQueue<'a> {
-    fn new(a: &'a crossbeam_channel::Receiver<SchedulablePayload>) -> Self {
+    fn new(channel: &'a crossbeam_channel::Receiver<SchedulablePayload>) -> Self {
+        Self {channel}
     }
 }
 

From 2ae9b79e03a05c7cee5dd0c0b628b0465a4a88f4 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Wed, 21 Sep 2022 13:28:02 +0900
Subject: [PATCH 2896/3199] save

---
 scheduler/src/lib.rs | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 5645fdb2541d8a..218abd52e3d9ce 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -841,12 +841,17 @@ impl TaskQueue {
 
 struct ChannelBackedTaskQueue<'a> {
     channel: &'a crossbeam_channel::Receiver<SchedulablePayload>,
+    buffered_task: Option<usize>,
 }
 
 impl<'a> ChannelBackedTaskQueue<'a> {
     fn new(channel: &'a crossbeam_channel::Receiver<SchedulablePayload>) -> Self {
         Self {channel}
     }
+
+    if buffer(&mut self, task: usize) {
+        self.buffered_task = task;
+    }
 }
 
 #[inline(never)]

From 1472f7f34fea86ac88e3abb15d85a90032045a6f Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Wed, 21 Sep 2022 13:28:29 +0900
Subject: [PATCH 2897/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 218abd52e3d9ce..66ed7fc435dada 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -846,7 +846,7 @@ struct ChannelBackedTaskQueue<'a> {
 
 impl<'a> ChannelBackedTaskQueue<'a> {
     fn new(channel: &'a crossbeam_channel::Receiver<SchedulablePayload>) -> Self {
-        Self {channel}
+        Self {channel, buffered_task: None}
     }
 
     if buffer(&mut self, task: usize) {

From 678be25358b9cb543080b560212940741cd6520a Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Wed, 21 Sep 2022 13:28:42 +0900
Subject: [PATCH 2898/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 66ed7fc435dada..65e413f6fba9bb 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -849,7 +849,7 @@ impl<'a> ChannelBackedTaskQueue<'a> {
         Self {channel, buffered_task: None}
     }
 
-    if buffer(&mut self, task: usize) {
+    fn buffer(&mut self, task: usize) {
         self.buffered_task = task;
     }
 }

From 4e5c903758d74d7a26d6ea6c0c50ff411b95aac5 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Wed, 21 Sep 2022 13:29:07 +0900
Subject: [PATCH 2899/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 65e413f6fba9bb..abbe68d764666e 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -849,7 +849,7 @@ impl<'a> ChannelBackedTaskQueue<'a> {
         Self {channel, buffered_task: None}
     }
 
-    fn buffer(&mut self, task: usize) {
+    fn buffer(&mut self, task: TaskInQueue) {
         self.buffered_task = task;
     }
 }

From b33250df205c4b1dc485ebbbb9db67a331180f1e Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Wed, 21 Sep 2022 13:29:24 +0900
Subject: [PATCH 2900/3199] save

---
 scheduler/src/lib.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index abbe68d764666e..6f3c36441dbadb 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -841,7 +841,7 @@ impl TaskQueue {
 
 struct ChannelBackedTaskQueue<'a> {
     channel: &'a crossbeam_channel::Receiver<SchedulablePayload>,
-    buffered_task: Option<usize>,
+    buffered_task: Option<TaskInQueue>,
 }
 
 impl<'a> ChannelBackedTaskQueue<'a> {
@@ -850,7 +850,7 @@ impl<'a> ChannelBackedTaskQueue<'a> {
     }
 
     fn buffer(&mut self, task: TaskInQueue) {
-        self.buffered_task = task;
+        self.buffered_task = Some(task);
     }
 }
 

From 5eb37fc637be37f21bc4620bcb765bf650f5676b Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Wed, 21 Sep 2022 13:30:41 +0900
Subject: [PATCH 2901/3199] save

---
 scheduler/src/lib.rs | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 6f3c36441dbadb..96a945dde10560 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1596,15 +1596,15 @@ impl ScheduleStage {
                        }
                    }
                    recv(from_prev) -> maybe_from => {
-                       if maybe_start_time.is_none() {
-                            info!("schedule_once:initial id_{:016x}", random_id);
-                           maybe_start_time = Some(std::time::Instant::now());
-                           last_time = maybe_start_time.clone();
-                       }
                        match maybe_from {
                            Ok(SchedulablePayload(Flushable::Payload(task))) => {
-                                //Self::register_runnable_task(task, runnable_queue, &mut sequence_time);
-                                channel_backed_runnable_queue.buffer(task);
+                               if maybe_start_time.is_none() {
+                                    info!("schedule_once:initial id_{:016x}", random_id);
+                                   maybe_start_time = Some(std::time::Instant::now());
+                                   last_time = maybe_start_time.clone();
+                               }
+                               //Self::register_runnable_task(task, runnable_queue, &mut sequence_time);
+                               channel_backed_runnable_queue.buffer(task);
                            },
                            Ok(SchedulablePayload(Flushable::Flush(checkpoint))) => {
                                assert_eq!(from_prev.len(), 0);

From 18c8e6dd5de3774069d1384426c8710a01f77bcd Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Wed, 21 Sep 2022 13:32:03 +0900
Subject: [PATCH 2902/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 96a945dde10560..0be8f5db4f1314 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1479,7 +1479,7 @@ impl ScheduleStage {
         ast: AST,
         random_id: u64,
         max_executing_queue_count: usize,
-        runnable_queue: &mut TaskQueue,
+        _runnable_queue: &mut TaskQueue,
         address_book: &mut AddressBook,
         mut from_prev: &'a crossbeam_channel::Receiver<SchedulablePayload>,
         to_execute_substage: &crossbeam_channel::Sender<ExecutablePayload>,

From 8d635ecd10bb9536556ed593af082c63d0610eed Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Wed, 21 Sep 2022 13:33:26 +0900
Subject: [PATCH 2903/3199] save

---
 scheduler/src/lib.rs | 24 ++++++++++++------------
 1 file changed, 12 insertions(+), 12 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 0be8f5db4f1314..2297890f7d12f9 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1589,7 +1589,7 @@ impl ScheduleStage {
                        } else {
                            assert_eq!(from_exec.len(), 0);
                            from_exec_disconnected = true;
-                           info!("flushing1..: {:?} {} {} {} {}", (from_disconnected, from_exec_disconnected), runnable_queue.task_count(), contended_count, executing_queue_count, provisioning_tracker_count);
+                           info!("flushing1..: {:?} {} {} {} {}", (from_disconnected, from_exec_disconnected), channel_backed_runnable_queue.task_count(), contended_count, executing_queue_count, provisioning_tracker_count);
                            if from_disconnected {
                                break;
                            }
@@ -1611,7 +1611,7 @@ impl ScheduleStage {
                                assert!(!from_disconnected);
                                from_disconnected = true;
                                from_prev = never;
-                               trace!("flushing2..: {:?} {} {} {} {}", (from_disconnected, from_exec_disconnected), runnable_queue.task_count(), contended_count, executing_queue_count, provisioning_tracker_count);
+                               trace!("flushing2..: {:?} {} {} {} {}", (from_disconnected, from_exec_disconnected), channel_backed_runnable_queue.task_count(), contended_count, executing_queue_count, provisioning_tracker_count);
                                assert!(maybe_checkpoint.is_none());
                                maybe_checkpoint = Some(checkpoint);
                            },
@@ -1621,7 +1621,7 @@ impl ScheduleStage {
                                assert!(maybe_checkpoint.is_none());
                                from_disconnected = true;
                                from_prev = never;
-                               trace!("flushing2..: {:?} {} {} {} {}", (from_disconnected, from_exec_disconnected), runnable_queue.task_count(), contended_count, executing_queue_count, provisioning_tracker_count);
+                               trace!("flushing2..: {:?} {} {} {} {}", (from_disconnected, from_exec_disconnected), channel_backed_runnable_queue.task_count(), contended_count, executing_queue_count, provisioning_tracker_count);
                            },
                        }
                    }
@@ -1632,7 +1632,7 @@ impl ScheduleStage {
             }
 
             no_more_work = from_disconnected
-                && runnable_queue.task_count()
+                && channel_backed_runnable_queue.task_count()
                     + contended_count
                     + executing_queue_count
                     + provisioning_tracker_count
@@ -1646,7 +1646,7 @@ impl ScheduleStage {
             let (mut from_len, mut from_exec_len) = (0, 0);
 
             loop {
-                let runnable_finished = from_disconnected && runnable_queue.has_no_task();
+                let runnable_finished = from_disconnected && channel_backed_runnable_queue.has_no_task();
 
                 let mut selection = TaskSelection::OnlyFromContended(if runnable_finished {
                     usize::max_value()
@@ -1659,7 +1659,7 @@ impl ScheduleStage {
                     let maybe_ee = Self::schedule_next_execution(
                         ast,
                         &task_sender,
-                        runnable_queue,
+                        channel_backed_runnable_queue,
                         address_book,
                         &mut contended_count,
                         prefer_immediate,
@@ -1677,7 +1677,7 @@ impl ScheduleStage {
                             .send(ExecutablePayload(ee))
                             .unwrap();
                     }
-                    debug!("schedule_once id_{:016x} [C] ch(prev: {}, exec: {}+{}|{}), r: {}, u/c: {}/{}, (imm+provi)/max: ({}+{})/{} s: {} l(s+f): {}+{}", random_id, (if from_disconnected { "-".to_string() } else { format!("{}", from_prev.len()) }), to_high_execute_substage.map(|t| format!("{}", t.len())).unwrap_or("-".into()), to_execute_substage.len(), from_exec.len(), runnable_queue.task_count(), address_book.uncontended_task_ids.len(), contended_count, executing_queue_count, provisioning_tracker_count, max_executing_queue_count, address_book.stuck_tasks.len(), processed_count, failed_lock_count);
+                    debug!("schedule_once id_{:016x} [C] ch(prev: {}, exec: {}+{}|{}), r: {}, u/c: {}/{}, (imm+provi)/max: ({}+{})/{} s: {} l(s+f): {}+{}", random_id, (if from_disconnected { "-".to_string() } else { format!("{}", from_prev.len()) }), to_high_execute_substage.map(|t| format!("{}", t.len())).unwrap_or("-".into()), to_execute_substage.len(), from_exec.len(), channel_backed_runnable_queue.task_count(), address_book.uncontended_task_ids.len(), contended_count, executing_queue_count, provisioning_tracker_count, max_executing_queue_count, address_book.stuck_tasks.len(), processed_count, failed_lock_count);
 
                     if no_aggresive_contended && !from_exec.is_empty() {
                         trace!("abort aggressive contended queue processing due to non-empty from_exec");
@@ -1690,14 +1690,14 @@ impl ScheduleStage {
                         if elapsed > std::time::Duration::from_millis(150) {
                             let delta = (processed_count - last_processed_count) as u128;
                             let elapsed2 = elapsed.as_micros();
-                            info!("schedule_once:interval id_{:016x} ch(prev: {}, exec: {}+{}|{}), r: {}, u/c: {}/{}, (imm+provi)/max: ({}+{})/{} s: {} l(s+f): {}+{} ({}txs/{}us={}tps)", random_id, (if from_disconnected { "-".to_string() } else { format!("{}", from_prev.len()) }), to_high_execute_substage.map(|t| format!("{}", t.len())).unwrap_or("-".into()), to_execute_substage.len(), from_exec.len(), runnable_queue.task_count(), address_book.uncontended_task_ids.len(), contended_count, executing_queue_count, provisioning_tracker_count, max_executing_queue_count, address_book.stuck_tasks.len(), processed_count, failed_lock_count, delta, elapsed.as_micros(), 1_000_000_u128*delta/elapsed2);
+                            info!("schedule_once:interval id_{:016x} ch(prev: {}, exec: {}+{}|{}), r: {}, u/c: {}/{}, (imm+provi)/max: ({}+{})/{} s: {} l(s+f): {}+{} ({}txs/{}us={}tps)", random_id, (if from_disconnected { "-".to_string() } else { format!("{}", from_prev.len()) }), to_high_execute_substage.map(|t| format!("{}", t.len())).unwrap_or("-".into()), to_execute_substage.len(), from_exec.len(), channel_backed_runnable_queue.task_count(), address_book.uncontended_task_ids.len(), contended_count, executing_queue_count, provisioning_tracker_count, max_executing_queue_count, address_book.stuck_tasks.len(), processed_count, failed_lock_count, delta, elapsed.as_micros(), 1_000_000_u128*delta/elapsed2);
                             (last_time, last_processed_count) =
                                 (Some(std::time::Instant::now()), processed_count);
                         }
                     }
                 }
                 let mut selection = TaskSelection::OnlyFromRunnable;
-                while !runnable_queue.has_no_task()
+                while !channel_backed_runnable_queue.has_no_task()
                     && selection.should_proceed()
                     && (to_high_execute_substage.is_some()
                         || executing_queue_count + provisioning_tracker_count
@@ -1708,7 +1708,7 @@ impl ScheduleStage {
                     let maybe_ee = Self::schedule_next_execution(
                         ast,
                         &task_sender,
-                        runnable_queue,
+                        channel_backed_runnable_queue,
                         address_book,
                         &mut contended_count,
                         prefer_immediate,
@@ -1723,7 +1723,7 @@ impl ScheduleStage {
                         executing_queue_count = executing_queue_count.checked_add(1).unwrap();
                         to_execute_substage.send(ExecutablePayload(ee)).unwrap();
                     }
-                    debug!("schedule_once id_{:016x} [R] ch(prev: {}, exec: {}+{}|{}), r: {}, u/c: {}/{}, (imm+provi)/max: ({}+{})/{} s: {} l(s+f): {}+{}", random_id, (if from_disconnected { "-".to_string() } else { format!("{}", from_prev.len()) }), to_high_execute_substage.map(|t| format!("{}", t.len())).unwrap_or("-".into()), to_execute_substage.len(), from_exec.len(), runnable_queue.task_count(), address_book.uncontended_task_ids.len(), contended_count, executing_queue_count, provisioning_tracker_count, max_executing_queue_count, address_book.stuck_tasks.len(), processed_count, failed_lock_count);
+                    debug!("schedule_once id_{:016x} [R] ch(prev: {}, exec: {}+{}|{}), r: {}, u/c: {}/{}, (imm+provi)/max: ({}+{})/{} s: {} l(s+f): {}+{}", random_id, (if from_disconnected { "-".to_string() } else { format!("{}", from_prev.len()) }), to_high_execute_substage.map(|t| format!("{}", t.len())).unwrap_or("-".into()), to_execute_substage.len(), from_exec.len(), channel_backed_runnable_queue.task_count(), address_book.uncontended_task_ids.len(), contended_count, executing_queue_count, provisioning_tracker_count, max_executing_queue_count, address_book.stuck_tasks.len(), processed_count, failed_lock_count);
 
                     if !from_exec.is_empty() {
                         trace!(
@@ -1738,7 +1738,7 @@ impl ScheduleStage {
                         if elapsed > std::time::Duration::from_millis(150) {
                             let delta = (processed_count - last_processed_count) as u128;
                             let elapsed2 = elapsed.as_micros();
-                            info!("schedule_once:interval id_{:016x} ch(prev: {}, exec: {}+{}|{}), r: {}, u/c: {}/{}, (imm+provi)/max: ({}+{})/{} s: {} l(s+f): {}+{} ({}txs/{}us={}tps)", random_id, (if from_disconnected { "-".to_string() } else { format!("{}", from_prev.len()) }), to_high_execute_substage.map(|t| format!("{}", t.len())).unwrap_or("-".into()), to_execute_substage.len(), from_exec.len(), runnable_queue.task_count(), address_book.uncontended_task_ids.len(), contended_count, executing_queue_count, provisioning_tracker_count, max_executing_queue_count, address_book.stuck_tasks.len(), processed_count, failed_lock_count, delta, elapsed.as_micros(), 1_000_000_u128*delta/elapsed2);
+                            info!("schedule_once:interval id_{:016x} ch(prev: {}, exec: {}+{}|{}), r: {}, u/c: {}/{}, (imm+provi)/max: ({}+{})/{} s: {} l(s+f): {}+{} ({}txs/{}us={}tps)", random_id, (if from_disconnected { "-".to_string() } else { format!("{}", from_prev.len()) }), to_high_execute_substage.map(|t| format!("{}", t.len())).unwrap_or("-".into()), to_execute_substage.len(), from_exec.len(), channel_backed_runnable_queue.task_count(), address_book.uncontended_task_ids.len(), contended_count, executing_queue_count, provisioning_tracker_count, max_executing_queue_count, address_book.stuck_tasks.len(), processed_count, failed_lock_count, delta, elapsed.as_micros(), 1_000_000_u128*delta/elapsed2);
                             (last_time, last_processed_count) =
                                 (Some(std::time::Instant::now()), processed_count);
                         }

From 810854fbab5827ee77e11d9b7a2fbfdca8e5e02c Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Wed, 21 Sep 2022 13:35:38 +0900
Subject: [PATCH 2904/3199] save

---
 scheduler/src/lib.rs | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 2297890f7d12f9..ae68ee6816ee4d 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -852,6 +852,17 @@ impl<'a> ChannelBackedTaskQueue<'a> {
     fn buffer(&mut self, task: TaskInQueue) {
         self.buffered_task = Some(task);
     }
+
+    fn task_count(&self) -> usize {
+        self.channel.len() + (match self.buffered_task {
+            None => 0,
+            Some(_) => 1,
+        })
+    }
+
+    fn has_no_task(&self) -> bool {
+        self.task_count() == 0
+    }
 }
 
 #[inline(never)]

From c9d55265b5072cba8174337376e86fd89abd8dfa Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Wed, 21 Sep 2022 13:36:09 +0900
Subject: [PATCH 2905/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index ae68ee6816ee4d..c56b4d665c9905 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1839,7 +1839,7 @@ impl ScheduleStage {
             } else {
                 "-".into()
             };
-            info!("schedule_once:final id_{:016x} (no_more_work: {}) ch(prev: {}, exec: {}|{}), runnnable: {}, contended: {}, (immediate+provisional)/max: ({}+{})/{} uncontended: {} stuck: {} miss: {}, overall: {}txs/{}us={}tps!", random_id, no_more_work, (if from_disconnected { "-".to_string() } else { format!("{}", from_prev.len()) }), to_execute_substage.len(), (if from_exec_disconnected { "-".to_string() } else { format!("{}", from_exec.len())}), runnable_queue.task_count(), contended_count, executing_queue_count, provisioning_tracker_count, max_executing_queue_count, address_book.uncontended_task_ids.len(), address_book.stuck_tasks.len(), failed_lock_count, processed_count, elapsed.as_micros(), tps_label);
+            info!("schedule_once:final id_{:016x} (no_more_work: {}) ch(prev: {}, exec: {}|{}), runnnable: {}, contended: {}, (immediate+provisional)/max: ({}+{})/{} uncontended: {} stuck: {} miss: {}, overall: {}txs/{}us={}tps!", random_id, no_more_work, (if from_disconnected { "-".to_string() } else { format!("{}", from_prev.len()) }), to_execute_substage.len(), (if from_exec_disconnected { "-".to_string() } else { format!("{}", from_exec.len())}), channel_backed_runnable_queue.task_count(), contended_count, executing_queue_count, provisioning_tracker_count, max_executing_queue_count, address_book.uncontended_task_ids.len(), address_book.stuck_tasks.len(), failed_lock_count, processed_count, elapsed.as_micros(), tps_label);
         }
 
         maybe_checkpoint

From 9d8888663fb0b5879415cb6f46824ddcadb64901 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Wed, 21 Sep 2022 13:37:13 +0900
Subject: [PATCH 2906/3199] save

---
 scheduler/src/lib.rs | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index c56b4d665c9905..1455ac156a89d3 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -969,7 +969,8 @@ impl ScheduleStage {
 
     #[inline(never)]
     fn select_next_task<'a>(
-        runnable_queue: &'a mut TaskQueue,
+        //runnable_queue: &'a mut TaskQueue,
+        runnable_queue: &'a mut ChannelBackedTaskQueue,
         address_book: &mut AddressBook,
         contended_count: &usize,
         task_selection: &mut TaskSelection,
@@ -1055,7 +1056,8 @@ impl ScheduleStage {
     fn pop_from_queue_then_lock<AST: AtScheduleThread>(
         ast: AST,
         task_sender: &crossbeam_channel::Sender<(TaskInQueue, Vec<LockAttempt>)>,
-        runnable_queue: &mut TaskQueue,
+        //runnable_queue: &mut TaskQueue,
+        runnable_queue: &mut ChannelBackedTaskQueue,
         address_book: &mut AddressBook,
         contended_count: &mut usize,
         prefer_immediate: bool,
@@ -1440,7 +1442,8 @@ impl ScheduleStage {
     fn schedule_next_execution<AST: AtScheduleThread>(
         ast: AST,
         task_sender: &crossbeam_channel::Sender<(TaskInQueue, Vec<LockAttempt>)>,
-        runnable_queue: &mut TaskQueue,
+        //runnable_queue: &mut TaskQueue,
+        runnable_queue: &mut ChannelBackedTaskQueue,
         address_book: &mut AddressBook,
         contended_count: &mut usize,
         prefer_immediate: bool,

From a95de5a3560485f709e0a26549ba7e6c4fc82755 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Wed, 21 Sep 2022 13:38:21 +0900
Subject: [PATCH 2907/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 1455ac156a89d3..16dd0fe0afd65d 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1722,7 +1722,7 @@ impl ScheduleStage {
                     let maybe_ee = Self::schedule_next_execution(
                         ast,
                         &task_sender,
-                        channel_backed_runnable_queue,
+                        &mut channel_backed_runnable_queue,
                         address_book,
                         &mut contended_count,
                         prefer_immediate,

From d1a659950f9cdf5b72a108dc73bfd0bf6a6b44aa Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Wed, 21 Sep 2022 13:38:29 +0900
Subject: [PATCH 2908/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 16dd0fe0afd65d..32c773c6c5adc8 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1673,7 +1673,7 @@ impl ScheduleStage {
                     let maybe_ee = Self::schedule_next_execution(
                         ast,
                         &task_sender,
-                        channel_backed_runnable_queue,
+                        &mut channel_backed_runnable_queue,
                         address_book,
                         &mut contended_count,
                         prefer_immediate,

From f89a055a21559c20b1aa3c39d997258bf6e84a4d Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Wed, 21 Sep 2022 13:39:50 +0900
Subject: [PATCH 2909/3199] save

---
 scheduler/src/lib.rs | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 32c773c6c5adc8..6648f754beb4e9 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -863,6 +863,13 @@ impl<'a> ChannelBackedTaskQueue<'a> {
     fn has_no_task(&self) -> bool {
         self.task_count() == 0
     }
+
+    #[inline(never)]
+    fn heaviest_entry_to_execute(&mut self) -> Option<TaskInQueue> {
+        // unblocking recv must have been gurantted to succeed at the time of this method
+        // invocation
+        self.channel.try_recv().unwrap()
+    }
 }
 
 #[inline(never)]

From ad7210a53b7d6fb25b680c5c8106effa6ae69d97 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Wed, 21 Sep 2022 13:41:15 +0900
Subject: [PATCH 2910/3199] save

---
 scheduler/src/lib.rs | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 6648f754beb4e9..cbdd34d1db7103 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -868,7 +868,10 @@ impl<'a> ChannelBackedTaskQueue<'a> {
     fn heaviest_entry_to_execute(&mut self) -> Option<TaskInQueue> {
         // unblocking recv must have been gurantted to succeed at the time of this method
         // invocation
-        self.channel.try_recv().unwrap()
+        match self.channel.try_recv().unwrap() {
+            SchedulablePayload::Flushable::Payload(task) => task,
+            SchedulablePayload::Flushable::Checkpoint => unreachable!(),
+        }
     }
 }
 

From ff2b85080983cf3967dc5234f660aa44078d6b4e Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Wed, 21 Sep 2022 13:41:44 +0900
Subject: [PATCH 2911/3199] save

---
 scheduler/src/lib.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index cbdd34d1db7103..2ad60b910fe0d4 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -869,8 +869,8 @@ impl<'a> ChannelBackedTaskQueue<'a> {
         // unblocking recv must have been gurantted to succeed at the time of this method
         // invocation
         match self.channel.try_recv().unwrap() {
-            SchedulablePayload::Flushable::Payload(task) => task,
-            SchedulablePayload::Flushable::Checkpoint => unreachable!(),
+            SchedulablePayload(Flushable::Payload(task)) => task,
+            SchedulablePayload(Flushable::Checkpoint) => unreachable!(),
         }
     }
 }

From 094e5bac61935c1983f575cd679423af05a8ef1f Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Wed, 21 Sep 2022 13:43:08 +0900
Subject: [PATCH 2912/3199] save

---
 scheduler/src/lib.rs | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 2ad60b910fe0d4..8f2f028893c2e3 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -865,16 +865,18 @@ impl<'a> ChannelBackedTaskQueue<'a> {
     }
 
     #[inline(never)]
-    fn heaviest_entry_to_execute(&mut self) -> Option<TaskInQueue> {
+    fn heaviest_entry_to_execute(&mut self) -> Option<ChannelBackedTaskQueueEntry> {
         // unblocking recv must have been gurantted to succeed at the time of this method
         // invocation
         match self.channel.try_recv().unwrap() {
-            SchedulablePayload(Flushable::Payload(task)) => task,
+            SchedulablePayload(Flushable::Payload(task)) => ChannelBackedTaskQueueEntry(task),
             SchedulablePayload(Flushable::Checkpoint) => unreachable!(),
         }
     }
 }
 
+struct ChannelBackedTaskQueueEntry(TaskInQueue);
+
 #[inline(never)]
 fn attempt_lock_for_execution<'a, AST: AtScheduleThread>(
     ast: AST,

From 69bf58e5f295bbbd14bc2da053576c7c2bf5fac8 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Wed, 21 Sep 2022 13:43:32 +0900
Subject: [PATCH 2913/3199] save

---
 scheduler/src/lib.rs | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 8f2f028893c2e3..6d2485b3f2d617 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -877,6 +877,12 @@ impl<'a> ChannelBackedTaskQueue<'a> {
 
 struct ChannelBackedTaskQueueEntry(TaskInQueue);
 
+impl ChannelBackedTaskQueue {
+    fn remove() {
+        // noop;
+    }
+}
+
 #[inline(never)]
 fn attempt_lock_for_execution<'a, AST: AtScheduleThread>(
     ast: AST,

From fa7b5c85ee3baaa04058061bdf669b7fa979ee9d Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Wed, 21 Sep 2022 13:43:43 +0900
Subject: [PATCH 2914/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 6d2485b3f2d617..c970248dd486c7 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -878,7 +878,7 @@ impl<'a> ChannelBackedTaskQueue<'a> {
 struct ChannelBackedTaskQueueEntry(TaskInQueue);
 
 impl ChannelBackedTaskQueue {
-    fn remove() {
+    fn remove(&self) {
         // noop;
     }
 }

From 6e8d956187b5ec1cfb8885bc41597877a85de8a9 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Wed, 21 Sep 2022 13:44:02 +0900
Subject: [PATCH 2915/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index c970248dd486c7..fa6d8571da2732 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -877,7 +877,7 @@ impl<'a> ChannelBackedTaskQueue<'a> {
 
 struct ChannelBackedTaskQueueEntry(TaskInQueue);
 
-impl ChannelBackedTaskQueue {
+impl ChannelBackedTaskQueueEntry {
     fn remove(&self) {
         // noop;
     }

From 65f75725ca53c66988f199007e42c3a6b6da1098 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Wed, 21 Sep 2022 13:44:32 +0900
Subject: [PATCH 2916/3199] save

---
 scheduler/src/lib.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index fa6d8571da2732..6e801219207c69 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -878,8 +878,8 @@ impl<'a> ChannelBackedTaskQueue<'a> {
 struct ChannelBackedTaskQueueEntry(TaskInQueue);
 
 impl ChannelBackedTaskQueueEntry {
-    fn remove(&self) {
-        // noop;
+    fn remove(self) -> TaskInQueue {
+        self.0
     }
 }
 

From 69aad79e992a97352c40aefc272c5bfc5db50433 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Wed, 21 Sep 2022 13:44:49 +0900
Subject: [PATCH 2917/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 6e801219207c69..6058861c310321 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -870,7 +870,7 @@ impl<'a> ChannelBackedTaskQueue<'a> {
         // invocation
         match self.channel.try_recv().unwrap() {
             SchedulablePayload(Flushable::Payload(task)) => ChannelBackedTaskQueueEntry(task),
-            SchedulablePayload(Flushable::Checkpoint) => unreachable!(),
+            SchedulablePayload(Flushable::Flush) => unreachable!(),
         }
     }
 }

From 8acda39f06ed0a7cc9057e0929ee49093f1b0803 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Wed, 21 Sep 2022 13:45:07 +0900
Subject: [PATCH 2918/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 6058861c310321..fbf8c5f4dfdc0c 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -869,7 +869,7 @@ impl<'a> ChannelBackedTaskQueue<'a> {
         // unblocking recv must have been gurantted to succeed at the time of this method
         // invocation
         match self.channel.try_recv().unwrap() {
-            SchedulablePayload(Flushable::Payload(task)) => ChannelBackedTaskQueueEntry(task),
+            SchedulablePayload(Flushable::Payload(task)) => Some(ChannelBackedTaskQueueEntry(task)),
             SchedulablePayload(Flushable::Flush) => unreachable!(),
         }
     }

From 4904baa62e2bf12edd92961c7080b93abbc60aba Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Wed, 21 Sep 2022 13:46:12 +0900
Subject: [PATCH 2919/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index fbf8c5f4dfdc0c..0dd9ee84a3566e 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -870,7 +870,7 @@ impl<'a> ChannelBackedTaskQueue<'a> {
         // invocation
         match self.channel.try_recv().unwrap() {
             SchedulablePayload(Flushable::Payload(task)) => Some(ChannelBackedTaskQueueEntry(task)),
-            SchedulablePayload(Flushable::Flush) => unreachable!(),
+            SchedulablePayload(Flushable::Flush) => { todo!("buffer flush and propagate back to the outermost loop") }
         }
     }
 }

From d9ebd1ca8a123e07c6eaae1e2099d9b91e70718c Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Wed, 21 Sep 2022 13:46:37 +0900
Subject: [PATCH 2920/3199] save

---
 scheduler/src/lib.rs | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 0dd9ee84a3566e..1fefd0b56461e2 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -881,6 +881,10 @@ impl ChannelBackedTaskQueueEntry {
     fn remove(self) -> TaskInQueue {
         self.0
     }
+
+    fn key(&self) -> UniqueWeight {
+        self.0.unique_weight()
+    }
 }
 
 #[inline(never)]

From 64d9f37021974b314eb582f89a29c8255f66ea0d Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Wed, 21 Sep 2022 13:46:48 +0900
Subject: [PATCH 2921/3199] save

---
 scheduler/src/lib.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 1fefd0b56461e2..65c3ce2e5b5df9 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -882,8 +882,8 @@ impl ChannelBackedTaskQueueEntry {
         self.0
     }
 
-    fn key(&self) -> UniqueWeight {
-        self.0.unique_weight()
+    fn key(&self) -> &UniqueWeight {
+        &self.0.unique_weight()
     }
 }
 

From bec9275c4e759703f843a5acd303a6857570fb2a Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Wed, 21 Sep 2022 13:46:55 +0900
Subject: [PATCH 2922/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 65c3ce2e5b5df9..df1e0e2bf59eb8 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -883,7 +883,7 @@ impl ChannelBackedTaskQueueEntry {
     }
 
     fn key(&self) -> &UniqueWeight {
-        &self.0.unique_weight()
+        &self.0.unique_weight
     }
 }
 

From b6e0ca25ce98df1b2937e46e4ddd0f2fb13fe09c Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Wed, 21 Sep 2022 13:47:08 +0900
Subject: [PATCH 2923/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index df1e0e2bf59eb8..fa8c60f187e3a0 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -870,7 +870,7 @@ impl<'a> ChannelBackedTaskQueue<'a> {
         // invocation
         match self.channel.try_recv().unwrap() {
             SchedulablePayload(Flushable::Payload(task)) => Some(ChannelBackedTaskQueueEntry(task)),
-            SchedulablePayload(Flushable::Flush) => { todo!("buffer flush and propagate back to the outermost loop") }
+            SchedulablePayload(Flushable::Flush(_)) => { todo!("buffer flush and propagate back to the outermost loop") }
         }
     }
 }

From eff84171c3c24067406dc63f72d916359dbdb826 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Wed, 21 Sep 2022 13:51:27 +0900
Subject: [PATCH 2924/3199] save

---
 scheduler/src/lib.rs | 18 ++++++++++++++----
 1 file changed, 14 insertions(+), 4 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index fa8c60f187e3a0..d2cd2b6f30bcdd 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -997,10 +997,18 @@ impl ScheduleStage {
         contended_count: &usize,
         task_selection: &mut TaskSelection,
     ) -> Option<(TaskSource, TaskInQueue)> {
-        match (
-            runnable_queue.heaviest_entry_to_execute(),
-            Self::get_heaviest_from_contended(address_book),
-        ) {
+        let selected_heaviest_tasks = match task_selection {
+            TaskSelection::OnlyFromRunnable => (
+                runnable_queue.heaviest_entry_to_execute(),
+                None,
+            ),
+            TaskSelection::OnlyFromContended(_) => (
+                None,
+                Self::get_heaviest_from_contended(address_book),
+            ),
+        };
+
+        match selected_heaviest_tasks {
             (Some(heaviest_runnable_entry), None) => {
                 trace!("select: runnable only");
                 if task_selection.runnable_exclusive() {
@@ -1020,6 +1028,8 @@ impl ScheduleStage {
                 }
             }
             (Some(heaviest_runnable_entry), Some(weight_from_contended)) => {
+                unreachable!("heaviest_entry_to_execute isn't idempotent....");
+
                 let weight_from_runnable = heaviest_runnable_entry.key();
                 let uw = weight_from_contended.key();
 

From 87eece29abfe423ffa25e70eee29d408f0b0c751 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Wed, 21 Sep 2022 13:52:24 +0900
Subject: [PATCH 2925/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index d2cd2b6f30bcdd..a4b64e5f4a4102 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -826,7 +826,7 @@ impl TaskQueue {
     }
 
     #[inline(never)]
-    fn heaviest_entry_to_execute(&mut self) -> Option<TaskQueueOccupiedEntry<'_>> {
+    fn heaviest_entry_to_execute(&self) -> Option<TaskQueueOccupiedEntry<'_>> {
         self.tasks.last_entry()
     }
 

From b18e15e6081e6de2a0489f412be92c33207d9d58 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Wed, 21 Sep 2022 13:53:30 +0900
Subject: [PATCH 2926/3199] save

---
 scheduler/src/lib.rs | 17 +++++++++++------
 1 file changed, 11 insertions(+), 6 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index a4b64e5f4a4102..02dbe8884518b9 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -826,7 +826,7 @@ impl TaskQueue {
     }
 
     #[inline(never)]
-    fn heaviest_entry_to_execute(&self) -> Option<TaskQueueOccupiedEntry<'_>> {
+    fn heaviest_entry_to_execute(&mut self) -> Option<TaskQueueOccupiedEntry<'_>> {
         self.tasks.last_entry()
     }
 
@@ -866,11 +866,16 @@ impl<'a> ChannelBackedTaskQueue<'a> {
 
     #[inline(never)]
     fn heaviest_entry_to_execute(&mut self) -> Option<ChannelBackedTaskQueueEntry> {
-        // unblocking recv must have been gurantted to succeed at the time of this method
-        // invocation
-        match self.channel.try_recv().unwrap() {
-            SchedulablePayload(Flushable::Payload(task)) => Some(ChannelBackedTaskQueueEntry(task)),
-            SchedulablePayload(Flushable::Flush(_)) => { todo!("buffer flush and propagate back to the outermost loop") }
+        match self.buffered_task.take() {
+            Some(task) => Some(ChannelBackedTaskQueueEntry(task)),
+            None => {
+                // unblocking recv must have been gurantted to succeed at the time of this method
+                // invocation
+                match self.channel.try_recv().unwrap() {
+                    SchedulablePayload(Flushable::Payload(task)) => Some(ChannelBackedTaskQueueEntry(task)),
+                    SchedulablePayload(Flushable::Flush(_)) => { todo!("buffer flush and propagate back to the outermost loop") }
+                }
+            }
         }
     }
 }

From 1d69aa5611d4163afd4350dad0117093c6a0bef1 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Wed, 21 Sep 2022 13:56:35 +0900
Subject: [PATCH 2927/3199] save

---
 scheduler/src/lib.rs | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 02dbe8884518b9..34f4f2aa84ab6a 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -842,11 +842,12 @@ impl TaskQueue {
 struct ChannelBackedTaskQueue<'a> {
     channel: &'a crossbeam_channel::Receiver<SchedulablePayload>,
     buffered_task: Option<TaskInQueue>,
+    buffered_flush: Option<usize>,
 }
 
 impl<'a> ChannelBackedTaskQueue<'a> {
     fn new(channel: &'a crossbeam_channel::Receiver<SchedulablePayload>) -> Self {
-        Self {channel, buffered_task: None}
+        Self {channel, buffered_task: None, buffered_flush: None}
     }
 
     fn buffer(&mut self, task: TaskInQueue) {
@@ -873,7 +874,7 @@ impl<'a> ChannelBackedTaskQueue<'a> {
                 // invocation
                 match self.channel.try_recv().unwrap() {
                     SchedulablePayload(Flushable::Payload(task)) => Some(ChannelBackedTaskQueueEntry(task)),
-                    SchedulablePayload(Flushable::Flush(_)) => { todo!("buffer flush and propagate back to the outermost loop") }
+                    SchedulablePayload(Flushable::Flush(f)) => { assert!(self.buffered_flush.is_none()); self.buffered_flush = Some(f) }
                 }
             }
         }

From 4461908390e737384ff790b6a4a164c8cb8ee677 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Wed, 21 Sep 2022 13:56:56 +0900
Subject: [PATCH 2928/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 34f4f2aa84ab6a..35ed1a12e75601 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -874,7 +874,7 @@ impl<'a> ChannelBackedTaskQueue<'a> {
                 // invocation
                 match self.channel.try_recv().unwrap() {
                     SchedulablePayload(Flushable::Payload(task)) => Some(ChannelBackedTaskQueueEntry(task)),
-                    SchedulablePayload(Flushable::Flush(f)) => { assert!(self.buffered_flush.is_none()); self.buffered_flush = Some(f) }
+                    SchedulablePayload(Flushable::Flush(f)) => { assert!(self.buffered_flush.is_none()); self.buffered_flush = Some(f); None }
                 }
             }
         }

From 8d9787f15ec8609179195fe4a14f683bd6662b6f Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Wed, 21 Sep 2022 13:57:20 +0900
Subject: [PATCH 2929/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 35ed1a12e75601..5cbf44f8284500 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -842,7 +842,7 @@ impl TaskQueue {
 struct ChannelBackedTaskQueue<'a> {
     channel: &'a crossbeam_channel::Receiver<SchedulablePayload>,
     buffered_task: Option<TaskInQueue>,
-    buffered_flush: Option<usize>,
+    buffered_flush: Option<std::sync::Arc<Checkpoint>>,
 }
 
 impl<'a> ChannelBackedTaskQueue<'a> {

From e34eb183602bbd4b1ae4f5d3938514185eef7ed6 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Wed, 21 Sep 2022 16:44:38 +0900
Subject: [PATCH 2930/3199] save

---
 scheduler/src/lib.rs | 39 +++++++++++++++++++++++++++------------
 1 file changed, 27 insertions(+), 12 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 5cbf44f8284500..b1cec5b19628a2 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -865,6 +865,10 @@ impl<'a> ChannelBackedTaskQueue<'a> {
         self.task_count() == 0
     }
 
+    fn take_buffered_flush(&mut self) -> Option<std::sync::Arc<Checkpoint>> {
+        self.buffered_flush.take()
+    }
+
     #[inline(never)]
     fn heaviest_entry_to_execute(&mut self) -> Option<ChannelBackedTaskQueueEntry> {
         match self.buffered_task.take() {
@@ -1731,11 +1735,6 @@ impl ScheduleStage {
                     }
                     debug!("schedule_once id_{:016x} [C] ch(prev: {}, exec: {}+{}|{}), r: {}, u/c: {}/{}, (imm+provi)/max: ({}+{})/{} s: {} l(s+f): {}+{}", random_id, (if from_disconnected { "-".to_string() } else { format!("{}", from_prev.len()) }), to_high_execute_substage.map(|t| format!("{}", t.len())).unwrap_or("-".into()), to_execute_substage.len(), from_exec.len(), channel_backed_runnable_queue.task_count(), address_book.uncontended_task_ids.len(), contended_count, executing_queue_count, provisioning_tracker_count, max_executing_queue_count, address_book.stuck_tasks.len(), processed_count, failed_lock_count);
 
-                    if no_aggresive_contended && !from_exec.is_empty() {
-                        trace!("abort aggressive contended queue processing due to non-empty from_exec");
-                        break;
-                    }
-
                     interval_count += 1;
                     if interval_count % 100 == 0 {
                         let elapsed = last_time.unwrap().elapsed();
@@ -1747,6 +1746,11 @@ impl ScheduleStage {
                                 (Some(std::time::Instant::now()), processed_count);
                         }
                     }
+
+                    if no_aggresive_contended && !from_exec.is_empty() {
+                        trace!("abort aggressive contended queue processing due to non-empty from_exec");
+                        break;
+                    }
                 }
                 let mut selection = TaskSelection::OnlyFromRunnable;
                 while !channel_backed_runnable_queue.has_no_task()
@@ -1777,13 +1781,6 @@ impl ScheduleStage {
                     }
                     debug!("schedule_once id_{:016x} [R] ch(prev: {}, exec: {}+{}|{}), r: {}, u/c: {}/{}, (imm+provi)/max: ({}+{})/{} s: {} l(s+f): {}+{}", random_id, (if from_disconnected { "-".to_string() } else { format!("{}", from_prev.len()) }), to_high_execute_substage.map(|t| format!("{}", t.len())).unwrap_or("-".into()), to_execute_substage.len(), from_exec.len(), channel_backed_runnable_queue.task_count(), address_book.uncontended_task_ids.len(), contended_count, executing_queue_count, provisioning_tracker_count, max_executing_queue_count, address_book.stuck_tasks.len(), processed_count, failed_lock_count);
 
-                    if !from_exec.is_empty() {
-                        trace!(
-                            "abort aggressive runnable queue processing due to non-empty from_exec"
-                        );
-                        break;
-                    }
-
                     interval_count += 1;
                     if interval_count % 100 == 0 {
                         let elapsed = last_time.unwrap().elapsed();
@@ -1795,6 +1792,24 @@ impl ScheduleStage {
                                 (Some(std::time::Instant::now()), processed_count);
                         }
                     }
+
+                    if let Some(checkpoint) = channel_backed_runnable_queue.take_buffered_flush() {
+                        assert!(empty_from);
+                        assert_eq!(from_prev.len(), 0);
+                        assert!(!from_disconnected);
+                        from_disconnected = true;
+                        from_prev = never;
+                        assert!(maybe_checkpoint.is_none());
+                        maybe_checkpoint = Some(checkpoint);
+                        break;
+                    }
+
+                    if !from_exec.is_empty() {
+                        trace!(
+                            "abort aggressive runnable queue processing due to non-empty from_exec"
+                        );
+                        break;
+                    }
                 }
 
                 // this is hand-written unrolled select loop to reduce sched_yield (futex) as much

From 322d9895a056b6b2f485bdfebdf34e81cd3f6859 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Wed, 21 Sep 2022 16:47:08 +0900
Subject: [PATCH 2931/3199] save

---
 scheduler/src/lib.rs | 26 +++++++++++++-------------
 1 file changed, 13 insertions(+), 13 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index b1cec5b19628a2..998a0bca09adc3 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -830,7 +830,7 @@ impl TaskQueue {
         self.tasks.last_entry()
     }
 
-    fn task_count(&self) -> usize {
+    fn task_count_hint(&self) -> usize {
         self.tasks.len()
     }
 
@@ -854,7 +854,7 @@ impl<'a> ChannelBackedTaskQueue<'a> {
         self.buffered_task = Some(task);
     }
 
-    fn task_count(&self) -> usize {
+    fn task_count_hint(&self) -> usize {
         self.channel.len() + (match self.buffered_task {
             None => 0,
             Some(_) => 1,
@@ -862,7 +862,7 @@ impl<'a> ChannelBackedTaskQueue<'a> {
     }
 
     fn has_no_task(&self) -> bool {
-        self.task_count() == 0
+        self.task_count_hint() == 0
     }
 
     fn take_buffered_flush(&mut self) -> Option<std::sync::Arc<Checkpoint>> {
@@ -1073,7 +1073,7 @@ impl ScheduleStage {
             (None, None) => {
                 trace!("select: none");
 
-                if false && runnable_queue.task_count() == 0 && /* *contended_count > 0 &&*/ address_book.stuck_tasks.len() > 0
+                if false && runnable_queue.task_count_hint() == 0 && /* *contended_count > 0 &&*/ address_book.stuck_tasks.len() > 0
                 {
                     trace!("handling stuck...");
                     let (stuck_task_id, task) = address_book.stuck_tasks.pop_first().unwrap();
@@ -1645,7 +1645,7 @@ impl ScheduleStage {
                        } else {
                            assert_eq!(from_exec.len(), 0);
                            from_exec_disconnected = true;
-                           info!("flushing1..: {:?} {} {} {} {}", (from_disconnected, from_exec_disconnected), channel_backed_runnable_queue.task_count(), contended_count, executing_queue_count, provisioning_tracker_count);
+                           info!("flushing1..: {:?} {} {} {} {}", (from_disconnected, from_exec_disconnected), channel_backed_runnable_queue.task_count_hint(), contended_count, executing_queue_count, provisioning_tracker_count);
                            if from_disconnected {
                                break;
                            }
@@ -1667,7 +1667,7 @@ impl ScheduleStage {
                                assert!(!from_disconnected);
                                from_disconnected = true;
                                from_prev = never;
-                               trace!("flushing2..: {:?} {} {} {} {}", (from_disconnected, from_exec_disconnected), channel_backed_runnable_queue.task_count(), contended_count, executing_queue_count, provisioning_tracker_count);
+                               trace!("flushing2..: {:?} {} {} {} {}", (from_disconnected, from_exec_disconnected), channel_backed_runnable_queue.task_count_hint(), contended_count, executing_queue_count, provisioning_tracker_count);
                                assert!(maybe_checkpoint.is_none());
                                maybe_checkpoint = Some(checkpoint);
                            },
@@ -1677,7 +1677,7 @@ impl ScheduleStage {
                                assert!(maybe_checkpoint.is_none());
                                from_disconnected = true;
                                from_prev = never;
-                               trace!("flushing2..: {:?} {} {} {} {}", (from_disconnected, from_exec_disconnected), channel_backed_runnable_queue.task_count(), contended_count, executing_queue_count, provisioning_tracker_count);
+                               trace!("flushing2..: {:?} {} {} {} {}", (from_disconnected, from_exec_disconnected), channel_backed_runnable_queue.task_count_hint(), contended_count, executing_queue_count, provisioning_tracker_count);
                            },
                        }
                    }
@@ -1688,7 +1688,7 @@ impl ScheduleStage {
             }
 
             no_more_work = from_disconnected
-                && channel_backed_runnable_queue.task_count()
+                && channel_backed_runnable_queue.task_count_hint()
                     + contended_count
                     + executing_queue_count
                     + provisioning_tracker_count
@@ -1733,7 +1733,7 @@ impl ScheduleStage {
                             .send(ExecutablePayload(ee))
                             .unwrap();
                     }
-                    debug!("schedule_once id_{:016x} [C] ch(prev: {}, exec: {}+{}|{}), r: {}, u/c: {}/{}, (imm+provi)/max: ({}+{})/{} s: {} l(s+f): {}+{}", random_id, (if from_disconnected { "-".to_string() } else { format!("{}", from_prev.len()) }), to_high_execute_substage.map(|t| format!("{}", t.len())).unwrap_or("-".into()), to_execute_substage.len(), from_exec.len(), channel_backed_runnable_queue.task_count(), address_book.uncontended_task_ids.len(), contended_count, executing_queue_count, provisioning_tracker_count, max_executing_queue_count, address_book.stuck_tasks.len(), processed_count, failed_lock_count);
+                    debug!("schedule_once id_{:016x} [C] ch(prev: {}, exec: {}+{}|{}), r: {}, u/c: {}/{}, (imm+provi)/max: ({}+{})/{} s: {} l(s+f): {}+{}", random_id, (if from_disconnected { "-".to_string() } else { format!("{}", from_prev.len()) }), to_high_execute_substage.map(|t| format!("{}", t.len())).unwrap_or("-".into()), to_execute_substage.len(), from_exec.len(), channel_backed_runnable_queue.task_count_hint(), address_book.uncontended_task_ids.len(), contended_count, executing_queue_count, provisioning_tracker_count, max_executing_queue_count, address_book.stuck_tasks.len(), processed_count, failed_lock_count);
 
                     interval_count += 1;
                     if interval_count % 100 == 0 {
@@ -1741,7 +1741,7 @@ impl ScheduleStage {
                         if elapsed > std::time::Duration::from_millis(150) {
                             let delta = (processed_count - last_processed_count) as u128;
                             let elapsed2 = elapsed.as_micros();
-                            info!("schedule_once:interval id_{:016x} ch(prev: {}, exec: {}+{}|{}), r: {}, u/c: {}/{}, (imm+provi)/max: ({}+{})/{} s: {} l(s+f): {}+{} ({}txs/{}us={}tps)", random_id, (if from_disconnected { "-".to_string() } else { format!("{}", from_prev.len()) }), to_high_execute_substage.map(|t| format!("{}", t.len())).unwrap_or("-".into()), to_execute_substage.len(), from_exec.len(), channel_backed_runnable_queue.task_count(), address_book.uncontended_task_ids.len(), contended_count, executing_queue_count, provisioning_tracker_count, max_executing_queue_count, address_book.stuck_tasks.len(), processed_count, failed_lock_count, delta, elapsed.as_micros(), 1_000_000_u128*delta/elapsed2);
+                            info!("schedule_once:interval id_{:016x} ch(prev: {}, exec: {}+{}|{}), r: {}, u/c: {}/{}, (imm+provi)/max: ({}+{})/{} s: {} l(s+f): {}+{} ({}txs/{}us={}tps)", random_id, (if from_disconnected { "-".to_string() } else { format!("{}", from_prev.len()) }), to_high_execute_substage.map(|t| format!("{}", t.len())).unwrap_or("-".into()), to_execute_substage.len(), from_exec.len(), channel_backed_runnable_queue.task_count_hint(), address_book.uncontended_task_ids.len(), contended_count, executing_queue_count, provisioning_tracker_count, max_executing_queue_count, address_book.stuck_tasks.len(), processed_count, failed_lock_count, delta, elapsed.as_micros(), 1_000_000_u128*delta/elapsed2);
                             (last_time, last_processed_count) =
                                 (Some(std::time::Instant::now()), processed_count);
                         }
@@ -1779,7 +1779,7 @@ impl ScheduleStage {
                         executing_queue_count = executing_queue_count.checked_add(1).unwrap();
                         to_execute_substage.send(ExecutablePayload(ee)).unwrap();
                     }
-                    debug!("schedule_once id_{:016x} [R] ch(prev: {}, exec: {}+{}|{}), r: {}, u/c: {}/{}, (imm+provi)/max: ({}+{})/{} s: {} l(s+f): {}+{}", random_id, (if from_disconnected { "-".to_string() } else { format!("{}", from_prev.len()) }), to_high_execute_substage.map(|t| format!("{}", t.len())).unwrap_or("-".into()), to_execute_substage.len(), from_exec.len(), channel_backed_runnable_queue.task_count(), address_book.uncontended_task_ids.len(), contended_count, executing_queue_count, provisioning_tracker_count, max_executing_queue_count, address_book.stuck_tasks.len(), processed_count, failed_lock_count);
+                    debug!("schedule_once id_{:016x} [R] ch(prev: {}, exec: {}+{}|{}), r: {}, u/c: {}/{}, (imm+provi)/max: ({}+{})/{} s: {} l(s+f): {}+{}", random_id, (if from_disconnected { "-".to_string() } else { format!("{}", from_prev.len()) }), to_high_execute_substage.map(|t| format!("{}", t.len())).unwrap_or("-".into()), to_execute_substage.len(), from_exec.len(), channel_backed_runnable_queue.task_count_hint(), address_book.uncontended_task_ids.len(), contended_count, executing_queue_count, provisioning_tracker_count, max_executing_queue_count, address_book.stuck_tasks.len(), processed_count, failed_lock_count);
 
                     interval_count += 1;
                     if interval_count % 100 == 0 {
@@ -1787,7 +1787,7 @@ impl ScheduleStage {
                         if elapsed > std::time::Duration::from_millis(150) {
                             let delta = (processed_count - last_processed_count) as u128;
                             let elapsed2 = elapsed.as_micros();
-                            info!("schedule_once:interval id_{:016x} ch(prev: {}, exec: {}+{}|{}), r: {}, u/c: {}/{}, (imm+provi)/max: ({}+{})/{} s: {} l(s+f): {}+{} ({}txs/{}us={}tps)", random_id, (if from_disconnected { "-".to_string() } else { format!("{}", from_prev.len()) }), to_high_execute_substage.map(|t| format!("{}", t.len())).unwrap_or("-".into()), to_execute_substage.len(), from_exec.len(), channel_backed_runnable_queue.task_count(), address_book.uncontended_task_ids.len(), contended_count, executing_queue_count, provisioning_tracker_count, max_executing_queue_count, address_book.stuck_tasks.len(), processed_count, failed_lock_count, delta, elapsed.as_micros(), 1_000_000_u128*delta/elapsed2);
+                            info!("schedule_once:interval id_{:016x} ch(prev: {}, exec: {}+{}|{}), r: {}, u/c: {}/{}, (imm+provi)/max: ({}+{})/{} s: {} l(s+f): {}+{} ({}txs/{}us={}tps)", random_id, (if from_disconnected { "-".to_string() } else { format!("{}", from_prev.len()) }), to_high_execute_substage.map(|t| format!("{}", t.len())).unwrap_or("-".into()), to_execute_substage.len(), from_exec.len(), channel_backed_runnable_queue.task_count_hint(), address_book.uncontended_task_ids.len(), contended_count, executing_queue_count, provisioning_tracker_count, max_executing_queue_count, address_book.stuck_tasks.len(), processed_count, failed_lock_count, delta, elapsed.as_micros(), 1_000_000_u128*delta/elapsed2);
                             (last_time, last_processed_count) =
                                 (Some(std::time::Instant::now()), processed_count);
                         }
@@ -1895,7 +1895,7 @@ impl ScheduleStage {
             } else {
                 "-".into()
             };
-            info!("schedule_once:final id_{:016x} (no_more_work: {}) ch(prev: {}, exec: {}|{}), runnnable: {}, contended: {}, (immediate+provisional)/max: ({}+{})/{} uncontended: {} stuck: {} miss: {}, overall: {}txs/{}us={}tps!", random_id, no_more_work, (if from_disconnected { "-".to_string() } else { format!("{}", from_prev.len()) }), to_execute_substage.len(), (if from_exec_disconnected { "-".to_string() } else { format!("{}", from_exec.len())}), channel_backed_runnable_queue.task_count(), contended_count, executing_queue_count, provisioning_tracker_count, max_executing_queue_count, address_book.uncontended_task_ids.len(), address_book.stuck_tasks.len(), failed_lock_count, processed_count, elapsed.as_micros(), tps_label);
+            info!("schedule_once:final id_{:016x} (no_more_work: {}) ch(prev: {}, exec: {}|{}), runnnable: {}, contended: {}, (immediate+provisional)/max: ({}+{})/{} uncontended: {} stuck: {} miss: {}, overall: {}txs/{}us={}tps!", random_id, no_more_work, (if from_disconnected { "-".to_string() } else { format!("{}", from_prev.len()) }), to_execute_substage.len(), (if from_exec_disconnected { "-".to_string() } else { format!("{}", from_exec.len())}), channel_backed_runnable_queue.task_count_hint(), contended_count, executing_queue_count, provisioning_tracker_count, max_executing_queue_count, address_book.uncontended_task_ids.len(), address_book.stuck_tasks.len(), failed_lock_count, processed_count, elapsed.as_micros(), tps_label);
         }
 
         maybe_checkpoint

From f9c8970cc1819e36e52f7a34b16f146f2cbd9614 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Wed, 21 Sep 2022 16:54:29 +0900
Subject: [PATCH 2932/3199] save

---
 scheduler/src/lib.rs | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 998a0bca09adc3..ac2fde2e158a06 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -834,7 +834,7 @@ impl TaskQueue {
         self.tasks.len()
     }
 
-    fn has_no_task(&self) -> bool {
+    fn has_no_task_hint(&self) -> bool {
         self.tasks.is_empty()
     }
 }
@@ -851,6 +851,7 @@ impl<'a> ChannelBackedTaskQueue<'a> {
     }
 
     fn buffer(&mut self, task: TaskInQueue) {
+        assert!(self.buffered_task.is_none());
         self.buffered_task = Some(task);
     }
 
@@ -861,7 +862,7 @@ impl<'a> ChannelBackedTaskQueue<'a> {
         })
     }
 
-    fn has_no_task(&self) -> bool {
+    fn has_no_task_hint(&self) -> bool {
         self.task_count_hint() == 0
     }
 
@@ -1073,7 +1074,7 @@ impl ScheduleStage {
             (None, None) => {
                 trace!("select: none");
 
-                if false && runnable_queue.task_count_hint() == 0 && /* *contended_count > 0 &&*/ address_book.stuck_tasks.len() > 0
+                if false && runnable_queue.has_no_task_hint() && /* *contended_count > 0 &&*/ address_book.stuck_tasks.len() > 0
                 {
                     trace!("handling stuck...");
                     let (stuck_task_id, task) = address_book.stuck_tasks.pop_first().unwrap();
@@ -1516,7 +1517,7 @@ impl ScheduleStage {
     }
 
     #[inline(never)]
-    fn register_runnable_task(
+    fn _register_runnable_task(
         weighted_tx: TaskInQueue,
         runnable_queue: &mut TaskQueue,
         sequence_time: &mut usize,
@@ -1702,7 +1703,7 @@ impl ScheduleStage {
             let (mut from_len, mut from_exec_len) = (0, 0);
 
             loop {
-                let runnable_finished = from_disconnected && channel_backed_runnable_queue.has_no_task();
+                let runnable_finished = from_disconnected && channel_backed_runnable_queue.has_no_task_hint();
 
                 let mut selection = TaskSelection::OnlyFromContended(if runnable_finished {
                     usize::max_value()
@@ -1753,7 +1754,7 @@ impl ScheduleStage {
                     }
                 }
                 let mut selection = TaskSelection::OnlyFromRunnable;
-                while !channel_backed_runnable_queue.has_no_task()
+                while !channel_backed_runnable_queue.has_no_task_hint()
                     && selection.should_proceed()
                     && (to_high_execute_substage.is_some()
                         || executing_queue_count + provisioning_tracker_count
@@ -1794,7 +1795,6 @@ impl ScheduleStage {
                     }
 
                     if let Some(checkpoint) = channel_backed_runnable_queue.take_buffered_flush() {
-                        assert!(empty_from);
                         assert_eq!(from_prev.len(), 0);
                         assert!(!from_disconnected);
                         from_disconnected = true;
@@ -1825,7 +1825,7 @@ impl ScheduleStage {
                         from_exec_len = from_exec.len();
                     }
                 }
-                (empty_from, empty_from_exec) = (from_len == 0, from_exec_len == 0);
+                (empty_from, empty_from_exec) = (true || from_len == 0, from_exec_len == 0);
 
                 if empty_from && empty_from_exec {
                     break;

From 1dc2702b6ffb099b5aa2d1756f91286eba8c47a3 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Wed, 21 Sep 2022 16:55:22 +0900
Subject: [PATCH 2933/3199] save

---
 scheduler/src/lib.rs | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index ac2fde2e158a06..cdff2706717ff3 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1848,6 +1848,8 @@ impl ScheduleStage {
                             .unwrap();
                     }
                     if !empty_from {
+                        unreachable!();
+
                         let SchedulablePayload(schedulable) = from_prev.recv().unwrap();
                         from_len = from_len.checked_sub(1).unwrap();
                         empty_from = from_len == 0;

From bd0f3024993423a04b4fbc82daa06471c499b564 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Wed, 21 Sep 2022 20:57:42 +0900
Subject: [PATCH 2934/3199] save

---
 scheduler/src/lib.rs | 74 +++++++++++++++++++++++---------------------
 1 file changed, 38 insertions(+), 36 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index cdff2706717ff3..1123f591b02d1f 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -75,42 +75,9 @@ impl ExecutionEnvironment {
             > 0;
         for lock_attempt in self.finalized_lock_attempts.iter_mut() {
             let contended_unique_weights = lock_attempt.target_contended_unique_weights();
-            contended_unique_weights
-                .heaviest_task_cursor()
-                .map(|mut task_cursor| {
-                    let mut found = true;
-                    let mut removed = false;
-                    let mut task = task_cursor.value();
-                    //task.trace_timestamps("in_exec(initial list)");
-                    assert!(!task.already_finished());
-                    while !task.currently_contended() {
-                        if task_cursor.key() == &uq {
-                            assert!(should_remove);
-                            removed = task_cursor.remove();
-                            assert!(removed);
-                        }
-                        if let Some(new_cursor) = task_cursor.prev() {
-                            assert!(new_cursor.key() < task_cursor.key());
-                            task_cursor = new_cursor;
-                            task = task_cursor.value();
-                            assert!(!task.already_finished());
-                            //task.trace_timestamps("in_exec(subsequent list)");
-                        } else {
-                            found = false;
-                            break;
-                        }
-                    }
-                    if should_remove && !removed {
-                        contended_unique_weights.remove_task(&uq);
-                    }
-                    found.then(|| Task::clone_in_queue(task))
-                })
-                .flatten()
-                .map(|task| {
-                    //task.trace_timestamps(&format!("in_exec(heaviest:{})", self.task.queue_time_label()));
-                    lock_attempt.heaviest_uncontended = Some(task);
-                    ()
-                });
+            if let Some(heaviest_uncontended) = contended_unique_weights.reindex(should_remove, uq) {
+                lock_attempt.heaviest_uncontended = heaviest_uncontended;
+            };
 
             if should_remove && lock_attempt.requested_usage == RequestedUsage::Writable {
                 lock_attempt
@@ -248,6 +215,41 @@ impl TaskIds {
     ) -> Option<crossbeam_skiplist::map::Entry<'_, UniqueWeight, TaskInQueue>> {
         self.task_ids.back()
     }
+
+    #[inline(never)]
+    fn reindex(&self, should_remove: bool, uq: UniqueWeight) -> Option<TaskInQueue> {
+        self
+            .heaviest_task_cursor()
+            .map(|mut task_cursor| {
+                let mut found = true;
+                let mut removed = false;
+                let mut task = task_cursor.value();
+                //task.trace_timestamps("in_exec(initial list)");
+                assert!(!task.already_finished());
+                while !task.currently_contended() {
+                    if task_cursor.key() == &uq {
+                        assert!(should_remove);
+                        removed = task_cursor.remove();
+                        assert!(removed);
+                    }
+                    if let Some(new_cursor) = task_cursor.prev() {
+                        assert!(new_cursor.key() < task_cursor.key());
+                        task_cursor = new_cursor;
+                        task = task_cursor.value();
+                        assert!(!task.already_finished());
+                        //task.trace_timestamps("in_exec(subsequent list)");
+                    } else {
+                        found = false;
+                        break;
+                    }
+                }
+                if should_remove && !removed {
+                    self.remove_task(&uq);
+                }
+                found.then(|| Task::clone_in_queue(task))
+            })
+            .flatten()
+    }
 }
 
 #[derive(Debug)]

From 93dc0d678cdd52e4baf2b48c3cc10bff02785150 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Wed, 21 Sep 2022 20:58:05 +0900
Subject: [PATCH 2935/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 1123f591b02d1f..ef5a2d13b0abb9 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -76,7 +76,7 @@ impl ExecutionEnvironment {
         for lock_attempt in self.finalized_lock_attempts.iter_mut() {
             let contended_unique_weights = lock_attempt.target_contended_unique_weights();
             if let Some(heaviest_uncontended) = contended_unique_weights.reindex(should_remove, uq) {
-                lock_attempt.heaviest_uncontended = heaviest_uncontended;
+                lock_attempt.heaviest_uncontended = Some(heaviest_uncontended);
             };
 
             if should_remove && lock_attempt.requested_usage == RequestedUsage::Writable {

From aa33b9913b556cb950a3bd2ac4a5673b008dee07 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Wed, 21 Sep 2022 21:00:39 +0900
Subject: [PATCH 2936/3199] save

---
 scheduler/src/lib.rs | 11 ++++++++---
 1 file changed, 8 insertions(+), 3 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index ef5a2d13b0abb9..b904f77a8b6dd4 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -216,6 +216,13 @@ impl TaskIds {
         self.task_ids.back()
     }
 
+    pub fn heaviest_task_id(&self) {
+        self
+            .task_ids
+            .back()
+            .map(|j| *j.key())
+    }
+
     #[inline(never)]
     fn reindex(&self, should_remove: bool, uq: UniqueWeight) -> Option<TaskInQueue> {
         self
@@ -343,9 +350,7 @@ impl AddressBook {
     ) -> CU {
         let tcuw = attempt
             .target_contended_unique_weights()
-            .task_ids
-            .back()
-            .map(|j| *j.key());
+            .heaviest_task_id()
 
         let strictly_lockable_for_replay = if tcuw.is_none() {
             true

From eb93acba6bd1fdb4bc5b436ffde704e1fa14d44c Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Wed, 21 Sep 2022 21:00:45 +0900
Subject: [PATCH 2937/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index b904f77a8b6dd4..8ca9eebac8c8cb 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -350,7 +350,7 @@ impl AddressBook {
     ) -> CU {
         let tcuw = attempt
             .target_contended_unique_weights()
-            .heaviest_task_id()
+            .heaviest_task_id();
 
         let strictly_lockable_for_replay = if tcuw.is_none() {
             true

From 8b1079e59401c26bee99bb72ee2f4f63f8de3d9a Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Wed, 21 Sep 2022 21:01:05 +0900
Subject: [PATCH 2938/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 8ca9eebac8c8cb..ec49b1bf691a7d 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -216,7 +216,7 @@ impl TaskIds {
         self.task_ids.back()
     }
 
-    pub fn heaviest_task_id(&self) {
+    pub fn heaviest_task_id(&self) -> &TaskId {
         self
             .task_ids
             .back()

From aa0f7c6d716ab930e3293e6f9fa9372b4fbc7d1d Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Wed, 21 Sep 2022 21:01:22 +0900
Subject: [PATCH 2939/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index ec49b1bf691a7d..d1b0254a5b243b 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -216,7 +216,7 @@ impl TaskIds {
         self.task_ids.back()
     }
 
-    pub fn heaviest_task_id(&self) -> &TaskId {
+    pub fn heaviest_task_id(&self) -> Option<TaskId> {
         self
             .task_ids
             .back()

From c9a4d3ed5867c5d4e23cfd1c58d2c5777024fa30 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Wed, 21 Sep 2022 21:01:52 +0900
Subject: [PATCH 2940/3199] save

---
 scheduler/src/lib.rs | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index d1b0254a5b243b..771e50469e329a 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -74,8 +74,7 @@ impl ExecutionEnvironment {
             .load(std::sync::atomic::Ordering::SeqCst)
             > 0;
         for lock_attempt in self.finalized_lock_attempts.iter_mut() {
-            let contended_unique_weights = lock_attempt.target_contended_unique_weights();
-            if let Some(heaviest_uncontended) = contended_unique_weights.reindex(should_remove, uq) {
+            if let Some(heaviest_uncontended) = lock_attempt.target_contended_unique_weights().reindex(should_remove, uq) {
                 lock_attempt.heaviest_uncontended = Some(heaviest_uncontended);
             };
 

From e3b6faef28d1d482020c2fa3e3a148821be9fa62 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Wed, 21 Sep 2022 21:04:36 +0900
Subject: [PATCH 2941/3199] save

---
 scheduler/src/lib.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 771e50469e329a..3ed6445740e77c 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -24,11 +24,11 @@ type PageRcInner = triomphe::Arc<(
     TaskIds,
     std::sync::atomic::AtomicUsize,
 )>;
+unsafe impl Send for PageRcInner {}
+unsafe impl Sync for PageRcInner {}
 
 #[derive(Debug, Clone)]
 pub struct PageRc(PageRcInner);
-unsafe impl Send for PageRc {}
-unsafe impl Sync for PageRc {}
 
 type CU = u64;
 

From 7adfa8c091f02fa49194e547230599792d040ae6 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Wed, 21 Sep 2022 21:04:43 +0900
Subject: [PATCH 2942/3199] save

---
 scheduler/src/lib.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 3ed6445740e77c..d75a34c20cbf2e 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -24,11 +24,11 @@ type PageRcInner = triomphe::Arc<(
     TaskIds,
     std::sync::atomic::AtomicUsize,
 )>;
-unsafe impl Send for PageRcInner {}
-unsafe impl Sync for PageRcInner {}
 
 #[derive(Debug, Clone)]
 pub struct PageRc(PageRcInner);
+unsafe impl Send for PageRcInner {}
+unsafe impl Sync for PageRcInner {}
 
 type CU = u64;
 

From bb24e81ddc3c3e981142598fcddeab32375120bf Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Wed, 21 Sep 2022 21:04:50 +0900
Subject: [PATCH 2943/3199] save

---
 scheduler/src/lib.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index d75a34c20cbf2e..771e50469e329a 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -27,8 +27,8 @@ type PageRcInner = triomphe::Arc<(
 
 #[derive(Debug, Clone)]
 pub struct PageRc(PageRcInner);
-unsafe impl Send for PageRcInner {}
-unsafe impl Sync for PageRcInner {}
+unsafe impl Send for PageRc {}
+unsafe impl Sync for PageRc {}
 
 type CU = u64;
 

From 497584370f7da9e19fd091329140158f4868a4f7 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Wed, 21 Sep 2022 21:05:44 +0900
Subject: [PATCH 2944/3199] save

---
 scheduler/src/lib.rs | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 771e50469e329a..40468ad516f856 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -21,7 +21,7 @@ unsafe impl Send for PageRc {}
 
 type PageRcInner = triomphe::Arc<(
     std::cell::RefCell<Page>,
-    TaskIds,
+    SkipListTaskIds,
     std::sync::atomic::AtomicUsize,
 )>;
 
@@ -186,12 +186,14 @@ pub enum RequestedUsage {
     Writable,
 }
 
+type TaskIds = SkipListTaskIds;
+
 #[derive(Debug, Default)]
-pub struct TaskIds {
+pub struct SkipListTaskIds {
     task_ids: crossbeam_skiplist::SkipMap<UniqueWeight, TaskInQueue>,
 }
 
-impl TaskIds {
+impl SkipListTaskIds {
     #[inline(never)]
     pub fn insert_task(&self, u: TaskId, task: TaskInQueue) {
         let mut is_inserted = false;

From 04e19fbfd4acca33d7e09713dffa4c3f582cfbfe Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Wed, 21 Sep 2022 21:07:05 +0900
Subject: [PATCH 2945/3199] save

---
 scheduler/src/lib.rs | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 40468ad516f856..3e746dffe0f949 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -74,7 +74,7 @@ impl ExecutionEnvironment {
             .load(std::sync::atomic::Ordering::SeqCst)
             > 0;
         for lock_attempt in self.finalized_lock_attempts.iter_mut() {
-            if let Some(heaviest_uncontended) = lock_attempt.target_contended_unique_weights().reindex(should_remove, uq) {
+            if let Some(heaviest_uncontended) = lock_attempt.target_contended_unique_weights().reindex(should_remove, &uq) {
                 lock_attempt.heaviest_uncontended = Some(heaviest_uncontended);
             };
 
@@ -225,7 +225,7 @@ impl SkipListTaskIds {
     }
 
     #[inline(never)]
-    fn reindex(&self, should_remove: bool, uq: UniqueWeight) -> Option<TaskInQueue> {
+    fn reindex(&self, should_remove: bool, uq: &UniqueWeight) -> Option<TaskInQueue> {
         self
             .heaviest_task_cursor()
             .map(|mut task_cursor| {
@@ -235,7 +235,7 @@ impl SkipListTaskIds {
                 //task.trace_timestamps("in_exec(initial list)");
                 assert!(!task.already_finished());
                 while !task.currently_contended() {
-                    if task_cursor.key() == &uq {
+                    if task_cursor.key() == uq {
                         assert!(should_remove);
                         removed = task_cursor.remove();
                         assert!(removed);
@@ -252,7 +252,7 @@ impl SkipListTaskIds {
                     }
                 }
                 if should_remove && !removed {
-                    self.remove_task(&uq);
+                    self.remove_task(uq);
                 }
                 found.then(|| Task::clone_in_queue(task))
             })

From abbca3d19310dd955bb9d47e5a4cdae95eede14d Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Wed, 21 Sep 2022 21:09:38 +0900
Subject: [PATCH 2946/3199] save

---
 scheduler/src/lib.rs | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 3e746dffe0f949..2409fd20a24a99 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -21,7 +21,7 @@ unsafe impl Send for PageRc {}
 
 type PageRcInner = triomphe::Arc<(
     std::cell::RefCell<Page>,
-    SkipListTaskIds,
+    //SkipListTaskIds,
     std::sync::atomic::AtomicUsize,
 )>;
 
@@ -267,6 +267,7 @@ pub struct Page {
     next_usage: Usage,
     provisional_task_ids: Vec<triomphe::Arc<ProvisioningTracker>>,
     cu: CU,
+    task_ids: TaskIds,
     //loaded account from Accounts db
     //comulative_cu for qos; i.e. track serialized cumulative keyed by addresses and bail out block
     //producing as soon as any one of cu from the executing thread reaches to the limit

From 427b20904bc1566dbe20d579a474a1ce2ee181b7 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Wed, 21 Sep 2022 21:10:15 +0900
Subject: [PATCH 2947/3199] save

---
 scheduler/src/lib.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 2409fd20a24a99..c0293b26ad04f2 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -147,11 +147,11 @@ impl LockAttempt {
     }
 
     pub fn target_contended_unique_weights(&self) -> &TaskIds {
-        &self.target.0 .1
+        panic!()//&self.target.0 .1
     }
 
     pub fn target_contended_write_task_count(&self) -> &std::sync::atomic::AtomicUsize {
-        &self.target.0 .2
+        &self.target.0.1
     }
 }
 

From cf9c64ed249d58190542e0dbd44b6919c185c255 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Wed, 21 Sep 2022 21:10:24 +0900
Subject: [PATCH 2948/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index c0293b26ad04f2..746a48ab1922a6 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -530,7 +530,7 @@ impl Preloader {
         PageRc::clone(&self.book.entry(address).or_insert_with(|| {
             PageRc(PageRcInner::new((
                 core::cell::RefCell::new(Page::new(&address, Usage::unused())),
-                Default::default(),
+                //Default::default(),
                 Default::default(),
             )))
         }))

From beb1e86ab722f4477f69269a05dae312142a307d Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Wed, 21 Sep 2022 21:10:41 +0900
Subject: [PATCH 2949/3199] save

---
 scheduler/src/lib.rs | 1 +
 1 file changed, 1 insertion(+)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 746a48ab1922a6..b96d4f0d3503d9 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -281,6 +281,7 @@ impl Page {
             next_usage: Usage::Unused,
             provisional_task_ids: Default::default(),
             cu: Default::default(),
+            task_ids: Default::default(),
         }
     }
 

From 15187dd15d2546421e0ecf9aa66df618dcd63344 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Wed, 21 Sep 2022 21:12:07 +0900
Subject: [PATCH 2950/3199] save

---
 scheduler/src/lib.rs | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index b96d4f0d3503d9..5f760f7c033bb8 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -153,6 +153,10 @@ impl LockAttempt {
     pub fn target_contended_write_task_count(&self) -> &std::sync::atomic::AtomicUsize {
         &self.target.0.1
     }
+
+    fn target_page_mut<AST: AtScheduleThread>(&self, _ast: AST) -> std::cell::RefMut<'_, Page> {
+        self.target.0.0.borrow_mut()
+    }
 }
 
 type UsageCount = usize;

From d1279d09d6f82621928fdbb01c4bd28146cd4b41 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Wed, 21 Sep 2022 21:12:45 +0900
Subject: [PATCH 2951/3199] save

---
 scheduler/src/lib.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 5f760f7c033bb8..e2db55a63665f1 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -154,8 +154,8 @@ impl LockAttempt {
         &self.target.0.1
     }
 
-    fn target_page_mut<AST: AtScheduleThread>(&self, _ast: AST) -> std::cell::RefMut<'_, Page> {
-        self.target.0.0.borrow_mut()
+    fn target_page_mut<AST: AtScheduleThread>(&self, ast: AST) -> std::cell::RefMut<'_, Page> {
+        self.target.page_mut(ast)
     }
 }
 

From b4737e0f83c59cc6705232012d9f5bddb22feb9d Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Wed, 21 Sep 2022 21:13:25 +0900
Subject: [PATCH 2952/3199] save

---
 scheduler/src/lib.rs | 15 +++++++--------
 1 file changed, 7 insertions(+), 8 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index e2db55a63665f1..2b76833231ec66 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -376,7 +376,7 @@ impl AddressBook {
 
         if !strictly_lockable_for_replay {
             attempt.status = LockStatus::Failed;
-            let page = attempt.target.page_mut(ast);
+            let page = attempt.target_page_mut(ast);
             return page.cu;
         }
 
@@ -386,7 +386,7 @@ impl AddressBook {
             status, /*, remembered*/
             ..
         } = attempt;
-        let mut page = target.page_mut(ast);
+        let mut page = target_page_mut(ast);
 
         let next_usage = page.next_usage;
         match page.current_usage {
@@ -473,7 +473,7 @@ impl AddressBook {
 
         let mut newly_uncontended = false;
 
-        let mut page = attempt.target.page_mut(ast);
+        let mut page = attempt.target_page_mut(ast);
 
         match &mut page.current_usage {
             Usage::Readonly(ref mut count) => match &attempt.requested_usage {
@@ -504,7 +504,7 @@ impl AddressBook {
 
     #[inline(never)]
     fn cancel<AST: AtScheduleThread>(&mut self, ast: AST, attempt: &mut LockAttempt) {
-        let mut page = attempt.target.page_mut(ast);
+        let mut page = attempt.target_page_mut(ast);
 
         match page.next_usage {
             Usage::Unused => {
@@ -942,7 +942,7 @@ fn attempt_lock_for_execution<'a, AST: AtScheduleThread>(
             LockStatus::Failed => {
                 trace!(
                     "lock failed: {}/{:?}",
-                    attempt.target.page_mut(ast).address_str,
+                    attempt.target_page_mut(ast).address_str,
                     attempt.requested_usage
                 );
                 unlockable_count += 1;
@@ -1317,8 +1317,7 @@ impl ScheduleStage {
         for l in next_task.lock_attempts_mut(ast).iter_mut() {
             match l.status {
                 LockStatus::Provisional => {
-                    l.target
-                        .page_mut(ast)
+                    l.target_page_mut(ast)
                         .provisional_task_ids
                         .push(triomphe::Arc::clone(&tracker));
                 }
@@ -1356,7 +1355,7 @@ impl ScheduleStage {
         for mut l in lock_attempts {
             let newly_uncontended = address_book.reset_lock(ast, &mut l, true);
 
-            let mut page = l.target.page_mut(ast);
+            let mut page = l.target_page_mut(ast);
             page.cu += cu;
             if newly_uncontended && page.next_usage == Usage::Unused {
                 //let mut inserted = false;

From 1cb2b4ce1b625436130647139ec93d53523a6809 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Wed, 21 Sep 2022 21:13:44 +0900
Subject: [PATCH 2953/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 2b76833231ec66..ad59ded027de6a 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -386,7 +386,7 @@ impl AddressBook {
             status, /*, remembered*/
             ..
         } = attempt;
-        let mut page = target_page_mut(ast);
+        let mut page = target.page_mut(ast);
 
         let next_usage = page.next_usage;
         match page.current_usage {

From d626af46008325e8823c5aaee2148b11c01598ab Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Wed, 21 Sep 2022 21:14:09 +0900
Subject: [PATCH 2954/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index ad59ded027de6a..67d9d87dbda3bb 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1355,7 +1355,7 @@ impl ScheduleStage {
         for mut l in lock_attempts {
             let newly_uncontended = address_book.reset_lock(ast, &mut l, true);
 
-            let mut page = l.target_page_mut(ast);
+            let mut page = l.target.page_mut(ast);
             page.cu += cu;
             if newly_uncontended && page.next_usage == Usage::Unused {
                 //let mut inserted = false;

From 5aa66384d4cbeab44b6dc628c2f68d94f0441f9c Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Wed, 21 Sep 2022 21:38:48 +0900
Subject: [PATCH 2955/3199] save

---
 scheduler/src/lib.rs | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 67d9d87dbda3bb..372348b057c051 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -146,9 +146,11 @@ impl LockAttempt {
         }
     }
 
+    /*
     pub fn target_contended_unique_weights(&self) -> &TaskIds {
         panic!()//&self.target.0 .1
     }
+    */
 
     pub fn target_contended_write_task_count(&self) -> &std::sync::atomic::AtomicUsize {
         &self.target.0.1
@@ -356,7 +358,8 @@ impl AddressBook {
         attempt: &mut LockAttempt,
     ) -> CU {
         let tcuw = attempt
-            .target_contended_unique_weights()
+            //.target_contended_unique_weights()
+            .target_page_mut(ast).contended_unique_weights.
             .heaviest_task_id();
 
         let strictly_lockable_for_replay = if tcuw.is_none() {

From 53d29ca21a2038bbf98f5284a7d45e7d7eee1103 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Wed, 21 Sep 2022 21:39:04 +0900
Subject: [PATCH 2956/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 372348b057c051..d48a47043c7bc3 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -359,7 +359,7 @@ impl AddressBook {
     ) -> CU {
         let tcuw = attempt
             //.target_contended_unique_weights()
-            .target_page_mut(ast).contended_unique_weights.
+            .target_page_mut(ast).contended_unique_weights
             .heaviest_task_id();
 
         let strictly_lockable_for_replay = if tcuw.is_none() {

From 140b2a81db92711db4b2939c298b0a6b8bc5c288 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Wed, 21 Sep 2022 21:39:14 +0900
Subject: [PATCH 2957/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index d48a47043c7bc3..41b1b5bc918dfa 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -359,7 +359,7 @@ impl AddressBook {
     ) -> CU {
         let tcuw = attempt
             //.target_contended_unique_weights()
-            .target_page_mut(ast).contended_unique_weights
+            .target_page_mut(ast).task_ids
             .heaviest_task_id();
 
         let strictly_lockable_for_replay = if tcuw.is_none() {

From 2d29251933e13286155c5aefb1e156b8fc4eef09 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Wed, 21 Sep 2022 21:40:03 +0900
Subject: [PATCH 2958/3199] save

---
 scheduler/src/lib.rs | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 41b1b5bc918dfa..950d2a78a9edc2 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -62,7 +62,7 @@ impl ExecutionEnvironment {
     //}
     //
     #[inline(never)]
-    pub fn reindex_with_address_book(&mut self) {
+    pub fn reindex_with_address_book<AST: AtScheduleThread>(&mut self, ast: AST) {
         assert!(!self.is_reindexed());
         self.is_reindexed = true;
 
@@ -74,7 +74,7 @@ impl ExecutionEnvironment {
             .load(std::sync::atomic::Ordering::SeqCst)
             > 0;
         for lock_attempt in self.finalized_lock_attempts.iter_mut() {
-            if let Some(heaviest_uncontended) = lock_attempt.target_contended_unique_weights().reindex(should_remove, &uq) {
+            if let Some(heaviest_uncontended) = lock_attempt.target_page_mut(ast).task_ids.reindex(should_remove, &uq) {
                 lock_attempt.heaviest_uncontended = Some(heaviest_uncontended);
             };
 
@@ -1468,7 +1468,7 @@ impl ScheduleStage {
     ) {
         // do par()-ly?
 
-        ee.reindex_with_address_book();
+        ee.reindex_with_address_book(ast);
         assert!(ee.is_reindexed());
 
         ee.task.record_commit_time(*commit_clock);

From ff3062abed040d27c1cae3d4f637fcc883489b4b Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Wed, 21 Sep 2022 21:40:28 +0900
Subject: [PATCH 2959/3199] save

---
 scheduler/src/lib.rs | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 950d2a78a9edc2..286376ba17ae19 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1624,9 +1624,11 @@ impl ScheduleStage {
                                 if task.already_finished() {
                                     break;
                                 }
+                                /*
                                 lock_attempt
                                     .target_contended_unique_weights()
                                     .insert_task(task.unique_weight, Task::clone_in_queue(&task));
+                                */
                                 todo!("contended_write_task_count!");
                             }
                         }

From f0627424af4fb89b009f2c13ef5370024e63e7b6 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Wed, 21 Sep 2022 21:40:52 +0900
Subject: [PATCH 2960/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 286376ba17ae19..e1067b004ca20e 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -784,7 +784,7 @@ impl Task {
     ) {
         for lock_attempt in &*this.lock_attempts_mut(ast) {
             lock_attempt
-                .target_contended_unique_weights()
+                .target_page_mut(ast).task_ids
                 .insert_task(this.unique_weight, Task::clone_in_queue(this));
 
             if lock_attempt.requested_usage == RequestedUsage::Writable {

From b7ec2a440d71c1bae1a2fc4b16da2b3f929f020d Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Wed, 21 Sep 2022 21:41:49 +0900
Subject: [PATCH 2961/3199] save

---
 scheduler/src/lib.rs | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index e1067b004ca20e..fd0959a6353901 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -74,7 +74,8 @@ impl ExecutionEnvironment {
             .load(std::sync::atomic::Ordering::SeqCst)
             > 0;
         for lock_attempt in self.finalized_lock_attempts.iter_mut() {
-            if let Some(heaviest_uncontended) = lock_attempt.target_page_mut(ast).task_ids.reindex(should_remove, &uq) {
+            let ll = lock_attempt.target_page_mut(ast).task_ids.reindex(should_remove, &uq);
+            if let Some(heaviest_uncontended) = ll {
                 lock_attempt.heaviest_uncontended = Some(heaviest_uncontended);
             };
 

From 642d9f3ccb64a8ead55873f4f864e3fea37a4a0f Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Wed, 21 Sep 2022 21:42:08 +0900
Subject: [PATCH 2962/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index fd0959a6353901..9068bc8ab0eab6 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -62,7 +62,7 @@ impl ExecutionEnvironment {
     //}
     //
     #[inline(never)]
-    pub fn reindex_with_address_book<AST: AtScheduleThread>(&mut self, ast: AST) {
+    fn reindex_with_address_book<AST: AtScheduleThread>(&mut self, ast: AST) {
         assert!(!self.is_reindexed());
         self.is_reindexed = true;
 

From 409f023d6d5a4beea1469923abaf0b8c514bb169 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Wed, 21 Sep 2022 21:43:20 +0900
Subject: [PATCH 2963/3199] save

---
 scheduler/src/lib.rs | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 9068bc8ab0eab6..80c147e7d2bded 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -193,13 +193,19 @@ pub enum RequestedUsage {
     Writable,
 }
 
-type TaskIds = SkipListTaskIds;
+//type TaskIds = SkipListTaskIds;
+type TaskIds = BtreeMapTaskIds;
 
 #[derive(Debug, Default)]
 pub struct SkipListTaskIds {
     task_ids: crossbeam_skiplist::SkipMap<UniqueWeight, TaskInQueue>,
 }
 
+#[derive(Debug, Default)]
+pub struct BTreeMapTaskIds {
+    task_ids: std::collections::BTreeMap<UniqueWeight, TaskInQueue>,
+}
+
 impl SkipListTaskIds {
     #[inline(never)]
     pub fn insert_task(&self, u: TaskId, task: TaskInQueue) {

From 69d01fe21ff0eca6cb293298302caa38371ae153 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Wed, 21 Sep 2022 21:43:40 +0900
Subject: [PATCH 2964/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 80c147e7d2bded..d49aa98a1e453d 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -194,7 +194,7 @@ pub enum RequestedUsage {
 }
 
 //type TaskIds = SkipListTaskIds;
-type TaskIds = BtreeMapTaskIds;
+type TaskIds = BTreeMapTaskIds;
 
 #[derive(Debug, Default)]
 pub struct SkipListTaskIds {

From 03c4b5b391e2c2e5ae18ee62eed0944869357b9a Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Wed, 21 Sep 2022 21:45:16 +0900
Subject: [PATCH 2965/3199] save

---
 scheduler/src/lib.rs | 63 ++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 63 insertions(+)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index d49aa98a1e453d..ef22b12b4e2476 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -273,6 +273,69 @@ impl SkipListTaskIds {
     }
 }
 
+impl BTreeMapTaskIds {
+    #[inline(never)]
+    pub fn insert_task(&self, u: TaskId, task: TaskInQueue) {
+        let pre_existed = self.task_ids.insert(u, task);
+        assert!(pre_existed.is_none()); //, "identical shouldn't exist: {:?}", unique_weight);
+    }
+
+    #[inline(never)]
+    pub fn remove_task(&self, u: &TaskId) {
+        let removed_entry = self.task_ids.remove(u);
+        assert!(removed_entry.is_some());
+    }
+
+    #[inline(never)]
+    pub fn heaviest_task_cursor(
+        &self,
+    ) -> Option<crossbeam_skiplist::map::Entry<'_, UniqueWeight, TaskInQueue>> {
+        self.task_ids.back()
+    }
+
+    pub fn heaviest_task_id(&self) -> Option<TaskId> {
+        self
+            .task_ids
+            .back()
+            .map(|j| *j.key())
+    }
+
+    #[inline(never)]
+    fn reindex(&self, should_remove: bool, uq: &UniqueWeight) -> Option<TaskInQueue> {
+        self
+            .heaviest_task_cursor()
+            .map(|mut task_cursor| {
+                let mut found = true;
+                let mut removed = false;
+                let mut task = task_cursor.value();
+                //task.trace_timestamps("in_exec(initial list)");
+                assert!(!task.already_finished());
+                while !task.currently_contended() {
+                    if task_cursor.key() == uq {
+                        assert!(should_remove);
+                        removed = task_cursor.remove();
+                        assert!(removed);
+                    }
+                    if let Some(new_cursor) = task_cursor.prev() {
+                        assert!(new_cursor.key() < task_cursor.key());
+                        task_cursor = new_cursor;
+                        task = task_cursor.value();
+                        assert!(!task.already_finished());
+                        //task.trace_timestamps("in_exec(subsequent list)");
+                    } else {
+                        found = false;
+                        break;
+                    }
+                }
+                if should_remove && !removed {
+                    self.remove_task(uq);
+                }
+                found.then(|| Task::clone_in_queue(task))
+            })
+            .flatten()
+    }
+}
+
 #[derive(Debug)]
 pub struct Page {
     address_str: String,

From 29b08dd650a32c18bcbb61672aaa3f70ed209457 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Wed, 21 Sep 2022 21:46:03 +0900
Subject: [PATCH 2966/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index ef22b12b4e2476..5c62b2196782f1 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -296,7 +296,7 @@ impl BTreeMapTaskIds {
     pub fn heaviest_task_id(&self) -> Option<TaskId> {
         self
             .task_ids
-            .back()
+            .last()
             .map(|j| *j.key())
     }
 

From c0e2f2e8ddb9847995b0e622a19f3c1f22cee29c Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Wed, 21 Sep 2022 21:46:49 +0900
Subject: [PATCH 2967/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 5c62b2196782f1..72da368943cf9b 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -296,7 +296,7 @@ impl BTreeMapTaskIds {
     pub fn heaviest_task_id(&self) -> Option<TaskId> {
         self
             .task_ids
-            .last()
+            .last_entry()
             .map(|j| *j.key())
     }
 

From 3ab2fd93b8caf956d6af4f9cef51b98a53564a4c Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Wed, 21 Sep 2022 21:47:05 +0900
Subject: [PATCH 2968/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 72da368943cf9b..77f99db82d8f9b 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -290,7 +290,7 @@ impl BTreeMapTaskIds {
     pub fn heaviest_task_cursor(
         &self,
     ) -> Option<crossbeam_skiplist::map::Entry<'_, UniqueWeight, TaskInQueue>> {
-        self.task_ids.back()
+        self.task_ids.last_entry()
     }
 
     pub fn heaviest_task_id(&self) -> Option<TaskId> {

From 558b1a7b58c889043c21f92efc08e69cfa97ed31 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Wed, 21 Sep 2022 21:47:29 +0900
Subject: [PATCH 2969/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 77f99db82d8f9b..cce8bcaeb7a19b 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -289,7 +289,7 @@ impl BTreeMapTaskIds {
     #[inline(never)]
     pub fn heaviest_task_cursor(
         &self,
-    ) -> Option<crossbeam_skiplist::map::Entry<'_, UniqueWeight, TaskInQueue>> {
+    ) -> Option<std::collections::btree_map::OccupiedEntry<'_, u64, triomphe::Arc<Task>>> {
         self.task_ids.last_entry()
     }
 

From bc02e732a1073c31041ad168744c40a863db4244 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Wed, 21 Sep 2022 21:48:19 +0900
Subject: [PATCH 2970/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index cce8bcaeb7a19b..cb97fefabd1959 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -290,7 +290,7 @@ impl BTreeMapTaskIds {
     pub fn heaviest_task_cursor(
         &self,
     ) -> Option<std::collections::btree_map::OccupiedEntry<'_, u64, triomphe::Arc<Task>>> {
-        self.task_ids.last_entry()
+        self.task_ids.riter()
     }
 
     pub fn heaviest_task_id(&self) -> Option<TaskId> {

From 7714ef1d80af0e919a3bcd0545c022c272a5f505 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Wed, 21 Sep 2022 21:48:31 +0900
Subject: [PATCH 2971/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index cb97fefabd1959..6e4d3c3efd1070 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -290,7 +290,7 @@ impl BTreeMapTaskIds {
     pub fn heaviest_task_cursor(
         &self,
     ) -> Option<std::collections::btree_map::OccupiedEntry<'_, u64, triomphe::Arc<Task>>> {
-        self.task_ids.riter()
+        self.task_ids.iter().rev()
     }
 
     pub fn heaviest_task_id(&self) -> Option<TaskId> {

From e3b2be01edd18df8fd7fb95cebb457c67d4f3b9a Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Wed, 21 Sep 2022 21:52:56 +0900
Subject: [PATCH 2972/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 6e4d3c3efd1070..cd0bb889aa269c 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -276,7 +276,7 @@ impl SkipListTaskIds {
 impl BTreeMapTaskIds {
     #[inline(never)]
     pub fn insert_task(&self, u: TaskId, task: TaskInQueue) {
-        let pre_existed = self.task_ids.insert(u, task);
+        let pre_existed = self.task_ids.insert(std::cmp::Reverse(u), task);
         assert!(pre_existed.is_none()); //, "identical shouldn't exist: {:?}", unique_weight);
     }
 

From 3553deda9d2b1236097e3d4c7ce91134c73f641d Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Wed, 21 Sep 2022 21:53:36 +0900
Subject: [PATCH 2973/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index cd0bb889aa269c..77e03d6cfb9aa7 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -203,7 +203,7 @@ pub struct SkipListTaskIds {
 
 #[derive(Debug, Default)]
 pub struct BTreeMapTaskIds {
-    task_ids: std::collections::BTreeMap<UniqueWeight, TaskInQueue>,
+    task_ids: std::collections::BTreeMap<std::cmp::Reverse<UniqueWeight>, TaskInQueue>,
 }
 
 impl SkipListTaskIds {

From b36fbc8d60137917c5171c47ea2ae29635680d4c Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Wed, 21 Sep 2022 21:54:08 +0900
Subject: [PATCH 2974/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 77e03d6cfb9aa7..1c2126ff4ae740 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -282,7 +282,7 @@ impl BTreeMapTaskIds {
 
     #[inline(never)]
     pub fn remove_task(&self, u: &TaskId) {
-        let removed_entry = self.task_ids.remove(u);
+        let removed_entry = self.task_ids.remove(std::cmp::Reverse(u));
         assert!(removed_entry.is_some());
     }
 

From a92d89e7ac8d2c89ba58027ac6f721987eab5f7a Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Wed, 21 Sep 2022 21:54:18 +0900
Subject: [PATCH 2975/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 1c2126ff4ae740..5e4f6e777fb7c5 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -282,7 +282,7 @@ impl BTreeMapTaskIds {
 
     #[inline(never)]
     pub fn remove_task(&self, u: &TaskId) {
-        let removed_entry = self.task_ids.remove(std::cmp::Reverse(u));
+        let removed_entry = self.task_ids.remove(&std::cmp::Reverse(u));
         assert!(removed_entry.is_some());
     }
 

From caff2d9745f43faed8c4eb12ff194481e06d1747 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Wed, 21 Sep 2022 21:54:28 +0900
Subject: [PATCH 2976/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 5e4f6e777fb7c5..227483750ccea2 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -282,7 +282,7 @@ impl BTreeMapTaskIds {
 
     #[inline(never)]
     pub fn remove_task(&self, u: &TaskId) {
-        let removed_entry = self.task_ids.remove(&std::cmp::Reverse(u));
+        let removed_entry = self.task_ids.remove(&std::cmp::Reverse(*u));
         assert!(removed_entry.is_some());
     }
 

From 5de806d79d2d9f87e092d5bb1ac5986d49837565 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Wed, 21 Sep 2022 21:55:05 +0900
Subject: [PATCH 2977/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 227483750ccea2..5cfa55441576ce 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -290,7 +290,7 @@ impl BTreeMapTaskIds {
     pub fn heaviest_task_cursor(
         &self,
     ) -> Option<std::collections::btree_map::OccupiedEntry<'_, u64, triomphe::Arc<Task>>> {
-        self.task_ids.iter().rev()
+        self.task_ids.iter()
     }
 
     pub fn heaviest_task_id(&self) -> Option<TaskId> {

From bd7cc8ff82a2b19cb13a2f9d2aa3a44da9b936d5 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Wed, 21 Sep 2022 21:59:21 +0900
Subject: [PATCH 2978/3199] save

---
 scheduler/src/lib.rs | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 5cfa55441576ce..6e4d3c3efd1070 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -203,7 +203,7 @@ pub struct SkipListTaskIds {
 
 #[derive(Debug, Default)]
 pub struct BTreeMapTaskIds {
-    task_ids: std::collections::BTreeMap<std::cmp::Reverse<UniqueWeight>, TaskInQueue>,
+    task_ids: std::collections::BTreeMap<UniqueWeight, TaskInQueue>,
 }
 
 impl SkipListTaskIds {
@@ -276,13 +276,13 @@ impl SkipListTaskIds {
 impl BTreeMapTaskIds {
     #[inline(never)]
     pub fn insert_task(&self, u: TaskId, task: TaskInQueue) {
-        let pre_existed = self.task_ids.insert(std::cmp::Reverse(u), task);
+        let pre_existed = self.task_ids.insert(u, task);
         assert!(pre_existed.is_none()); //, "identical shouldn't exist: {:?}", unique_weight);
     }
 
     #[inline(never)]
     pub fn remove_task(&self, u: &TaskId) {
-        let removed_entry = self.task_ids.remove(&std::cmp::Reverse(*u));
+        let removed_entry = self.task_ids.remove(u);
         assert!(removed_entry.is_some());
     }
 
@@ -290,7 +290,7 @@ impl BTreeMapTaskIds {
     pub fn heaviest_task_cursor(
         &self,
     ) -> Option<std::collections::btree_map::OccupiedEntry<'_, u64, triomphe::Arc<Task>>> {
-        self.task_ids.iter()
+        self.task_ids.iter().rev()
     }
 
     pub fn heaviest_task_id(&self) -> Option<TaskId> {

From 7e30df746b9882b7af4796268e22aa648c0ef1a8 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Wed, 21 Sep 2022 21:59:54 +0900
Subject: [PATCH 2979/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 6e4d3c3efd1070..eda2fb9dee0136 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -289,7 +289,7 @@ impl BTreeMapTaskIds {
     #[inline(never)]
     pub fn heaviest_task_cursor(
         &self,
-    ) -> Option<std::collections::btree_map::OccupiedEntry<'_, u64, triomphe::Arc<Task>>> {
+    ) -> impl Iterator<Item = TaskInQueue> {
         self.task_ids.iter().rev()
     }
 

From 68a29bbd382a71b95a215cfc447fe5cc73e11b2f Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Wed, 21 Sep 2022 22:00:41 +0900
Subject: [PATCH 2980/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index eda2fb9dee0136..ff532434754e3b 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -290,7 +290,7 @@ impl BTreeMapTaskIds {
     pub fn heaviest_task_cursor(
         &self,
     ) -> impl Iterator<Item = TaskInQueue> {
-        self.task_ids.iter().rev()
+        self.task_ids.values().rev()
     }
 
     pub fn heaviest_task_id(&self) -> Option<TaskId> {

From 43ebe7232cf9a8d33330bd0d46deef72481aff0d Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Wed, 21 Sep 2022 22:00:57 +0900
Subject: [PATCH 2981/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index ff532434754e3b..231dfb59dc771f 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -289,7 +289,7 @@ impl BTreeMapTaskIds {
     #[inline(never)]
     pub fn heaviest_task_cursor(
         &self,
-    ) -> impl Iterator<Item = TaskInQueue> {
+    ) -> impl Iterator<Item = &TaskInQueue> {
         self.task_ids.values().rev()
     }
 

From f815737127a5d30141d1b667d1eda1a7c0aee242 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Wed, 21 Sep 2022 22:07:52 +0900
Subject: [PATCH 2982/3199] save

---
 scheduler/src/lib.rs | 37 ++++++++-----------------------------
 1 file changed, 8 insertions(+), 29 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 231dfb59dc771f..8070a429ea7ead 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -224,7 +224,7 @@ impl SkipListTaskIds {
     }
 
     #[inline(never)]
-    pub fn heaviest_task_cursor(
+    fn heaviest_task_cursor(
         &self,
     ) -> Option<crossbeam_skiplist::map::Entry<'_, UniqueWeight, TaskInQueue>> {
         self.task_ids.back()
@@ -287,7 +287,7 @@ impl BTreeMapTaskIds {
     }
 
     #[inline(never)]
-    pub fn heaviest_task_cursor(
+    fn heaviest_task_cursor(
         &self,
     ) -> impl Iterator<Item = &TaskInQueue> {
         self.task_ids.values().rev()
@@ -302,37 +302,16 @@ impl BTreeMapTaskIds {
 
     #[inline(never)]
     fn reindex(&self, should_remove: bool, uq: &UniqueWeight) -> Option<TaskInQueue> {
+        if should_remove {
+            self.remove_task(uq);
+        }
+
         self
             .heaviest_task_cursor()
-            .map(|mut task_cursor| {
-                let mut found = true;
-                let mut removed = false;
-                let mut task = task_cursor.value();
-                //task.trace_timestamps("in_exec(initial list)");
+            .find(|task| {
                 assert!(!task.already_finished());
-                while !task.currently_contended() {
-                    if task_cursor.key() == uq {
-                        assert!(should_remove);
-                        removed = task_cursor.remove();
-                        assert!(removed);
-                    }
-                    if let Some(new_cursor) = task_cursor.prev() {
-                        assert!(new_cursor.key() < task_cursor.key());
-                        task_cursor = new_cursor;
-                        task = task_cursor.value();
-                        assert!(!task.already_finished());
-                        //task.trace_timestamps("in_exec(subsequent list)");
-                    } else {
-                        found = false;
-                        break;
-                    }
-                }
-                if should_remove && !removed {
-                    self.remove_task(uq);
-                }
-                found.then(|| Task::clone_in_queue(task))
+                task.currently_contended()
             })
-            .flatten()
     }
 }
 

From 9f4c005f519bc32faeb74fc018734ef6d367ab8a Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Wed, 21 Sep 2022 22:08:32 +0900
Subject: [PATCH 2983/3199] save

---
 scheduler/src/lib.rs | 1 +
 1 file changed, 1 insertion(+)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 8070a429ea7ead..0b2abff5e217df 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -312,6 +312,7 @@ impl BTreeMapTaskIds {
                 assert!(!task.already_finished());
                 task.currently_contended()
             })
+            .map(|task| Task::clone_in_queue(task))
     }
 }
 

From 1335f13dfbfb696042d631d0ee460049d1329cb8 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Wed, 21 Sep 2022 22:08:48 +0900
Subject: [PATCH 2984/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 0b2abff5e217df..fa976ee4e39d38 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -301,7 +301,7 @@ impl BTreeMapTaskIds {
     }
 
     #[inline(never)]
-    fn reindex(&self, should_remove: bool, uq: &UniqueWeight) -> Option<TaskInQueue> {
+    fn reindex(&mut self, should_remove: bool, uq: &UniqueWeight) -> Option<TaskInQueue> {
         if should_remove {
             self.remove_task(uq);
         }

From 658e20386a67cf8b1492900666bfa0dcae258fe5 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Wed, 21 Sep 2022 22:09:01 +0900
Subject: [PATCH 2985/3199] save

---
 scheduler/src/lib.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index fa976ee4e39d38..50a29a842fe43e 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -275,13 +275,13 @@ impl SkipListTaskIds {
 
 impl BTreeMapTaskIds {
     #[inline(never)]
-    pub fn insert_task(&self, u: TaskId, task: TaskInQueue) {
+    pub fn insert_task(&mut self, u: TaskId, task: TaskInQueue) {
         let pre_existed = self.task_ids.insert(u, task);
         assert!(pre_existed.is_none()); //, "identical shouldn't exist: {:?}", unique_weight);
     }
 
     #[inline(never)]
-    pub fn remove_task(&self, u: &TaskId) {
+    pub fn remove_task(&mut self, u: &TaskId) {
         let removed_entry = self.task_ids.remove(u);
         assert!(removed_entry.is_some());
     }

From d3f480f18f070b0ffa522d9097ed8a22525957de Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Wed, 21 Sep 2022 22:09:27 +0900
Subject: [PATCH 2986/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 50a29a842fe43e..624a52a4f689a0 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -293,7 +293,7 @@ impl BTreeMapTaskIds {
         self.task_ids.values().rev()
     }
 
-    pub fn heaviest_task_id(&self) -> Option<TaskId> {
+    pub fn heaviest_task_id(&mut self) -> Option<TaskId> {
         self
             .task_ids
             .last_entry()

From 933a00b4691f26bbcece8df5d9fb7f0cb0543593 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Wed, 21 Sep 2022 23:15:07 +0900
Subject: [PATCH 2987/3199] save

---
 runtime/src/bank.rs  |  2 +-
 scheduler/src/lib.rs | 20 ++++++++++----------
 2 files changed, 11 insertions(+), 11 deletions(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index c3d680a6069350..aad9a661991942 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1398,7 +1398,7 @@ impl Scheduler {
                 ee.execution_us = wall_time.as_us();
 
                 //ee.reindex_with_address_book();
-                processed_ee_sender.send(solana_scheduler::UnlockablePayload(ee)).unwrap();
+                processed_ee_sender.send(solana_scheduler::UnlockablePayload(ee, timings)).unwrap();
             }
             todo!();
 
diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 624a52a4f689a0..efdfaa8a74f785 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1598,7 +1598,7 @@ impl ScheduleStage {
     }
 
     #[must_use]
-    fn _run<'a, AST: AtScheduleThread>(
+    fn _run<'a, AST: AtScheduleThread, T>(
         ast: AST,
         random_id: u64,
         max_executing_queue_count: usize,
@@ -1607,8 +1607,8 @@ impl ScheduleStage {
         mut from_prev: &'a crossbeam_channel::Receiver<SchedulablePayload>,
         to_execute_substage: &crossbeam_channel::Sender<ExecutablePayload>,
         to_high_execute_substage: Option<&crossbeam_channel::Sender<ExecutablePayload>>,
-        from_exec: &crossbeam_channel::Receiver<UnlockablePayload>,
-        maybe_to_next_stage: Option<&crossbeam_channel::Sender<ExaminablePayload>>, // assume nonblocking
+        from_exec: &crossbeam_channel::Receiver<UnlockablePayload<T>>,
+        maybe_to_next_stage: Option<&crossbeam_channel::Sender<ExaminablePayload<T>>>, // assume nonblocking
         never: &'a crossbeam_channel::Receiver<SchedulablePayload>,
     ) -> Option<std::sync::Arc<Checkpoint>> {
         let mut maybe_start_time = None;
@@ -1706,11 +1706,11 @@ impl ScheduleStage {
             if !from_disconnected || executing_queue_count >= 1 {
                 crossbeam_channel::select! {
                    recv(from_exec) -> maybe_from_exec => {
-                       if let Ok(UnlockablePayload(mut processed_execution_environment)) = maybe_from_exec {
+                       if let Ok(UnlockablePayload((mut processed_execution_environment, extra))) = maybe_from_exec {
                            executing_queue_count = executing_queue_count.checked_sub(1).unwrap();
                            processed_count = processed_count.checked_add(1).unwrap();
                            Self::commit_processed_execution(ast, &mut processed_execution_environment, address_book, &mut commit_clock, &mut provisioning_tracker_count);
-                           to_next_stage.send(ExaminablePayload(processed_execution_environment)).unwrap();
+                           to_next_stage.send(ExaminablePayload(processed_execution_environment, extra)).unwrap();
                        } else {
                            assert_eq!(from_exec.len(), 0);
                            from_exec_disconnected = true;
@@ -1972,7 +1972,7 @@ impl ScheduleStage {
     }
 
     #[must_use]
-    pub fn run(
+    pub fn run<T>(
         random_id: u64,
         max_executing_queue_count: usize,
         runnable_queue: &mut TaskQueue,
@@ -1980,8 +1980,8 @@ impl ScheduleStage {
         from: &crossbeam_channel::Receiver<SchedulablePayload>,
         to_execute_substage: &crossbeam_channel::Sender<ExecutablePayload>,
         to_high_execute_substage: Option<&crossbeam_channel::Sender<ExecutablePayload>>,
-        from_execute_substage: &crossbeam_channel::Receiver<UnlockablePayload>,
-        maybe_to_next_stage: Option<&crossbeam_channel::Sender<ExaminablePayload>>, // assume nonblocking
+        from_execute_substage: &crossbeam_channel::Receiver<UnlockablePayload<T>>,
+        maybe_to_next_stage: Option<&crossbeam_channel::Sender<ExaminablePayload<T>>>, // assume nonblocking
     ) -> Option<std::sync::Arc<Checkpoint>> {
         #[derive(Clone, Copy, Debug)]
         struct AtTopOfScheduleThread;
@@ -2005,8 +2005,8 @@ impl ScheduleStage {
 
 pub struct SchedulablePayload(pub Flushable<TaskInQueue>);
 pub struct ExecutablePayload(pub Box<ExecutionEnvironment>);
-pub struct UnlockablePayload(pub Box<ExecutionEnvironment>);
-pub struct ExaminablePayload(pub Box<ExecutionEnvironment>);
+pub struct UnlockablePayload<T>(pub Box<ExecutionEnvironment>, T);
+pub struct ExaminablePayload<T>(pub Box<ExecutionEnvironment>, T);
 
 pub struct Checkpoint(std::sync::Mutex<usize>, std::sync::Condvar);
 

From 7b5fb9e8d7dc6dda1ac899ca7a948da28220e238 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Wed, 21 Sep 2022 23:15:42 +0900
Subject: [PATCH 2988/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index efdfaa8a74f785..023e3bfad895ee 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1628,7 +1628,7 @@ impl ScheduleStage {
 
         assert!(max_executing_queue_count > 0);
 
-        let (ee_sender, ee_receiver) = crossbeam_channel::unbounded::<ExaminablePayload>();
+        let (ee_sender, ee_receiver) = crossbeam_channel::unbounded::<ExaminablePayload<T>>();
 
         let (to_next_stage, maybe_reaper_thread_handle) =
             if let Some(to_next_stage) = maybe_to_next_stage {

From 4ecaed25b9c049d4cc04ffb7232a8ae3e883c621 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Wed, 21 Sep 2022 23:16:11 +0900
Subject: [PATCH 2989/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 023e3bfad895ee..c5b62240c978cb 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1643,7 +1643,7 @@ impl ScheduleStage {
                         unsafe impl NotAtScheduleThread for NotAtTopOfScheduleThread {}
                         let nast = NotAtTopOfScheduleThread;
 
-                        while let Ok(ExaminablePayload(a)) = ee_receiver.recv() {
+                        while let Ok(ExaminablePayload(a, _e)) = ee_receiver.recv() {
                             assert!(a.task.lock_attempts_not_mut(nast).is_empty());
                             //assert!(a.task.sequence_time() != usize::max_value());
                             //let lock_attempts = std::mem::take(&mut a.lock_attempts);

From cdbcee2d3bf0698947c1fb17ceb389b7b5dc1909 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Wed, 21 Sep 2022 23:16:30 +0900
Subject: [PATCH 2990/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index c5b62240c978cb..575897da497d29 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1706,7 +1706,7 @@ impl ScheduleStage {
             if !from_disconnected || executing_queue_count >= 1 {
                 crossbeam_channel::select! {
                    recv(from_exec) -> maybe_from_exec => {
-                       if let Ok(UnlockablePayload((mut processed_execution_environment, extra))) = maybe_from_exec {
+                       if let Ok(UnlockablePayload(mut processed_execution_environment, extra)) = maybe_from_exec {
                            executing_queue_count = executing_queue_count.checked_sub(1).unwrap();
                            processed_count = processed_count.checked_add(1).unwrap();
                            Self::commit_processed_execution(ast, &mut processed_execution_environment, address_book, &mut commit_clock, &mut provisioning_tracker_count);

From cfc53390de075eb9052286a48ccc9c5f7524ea12 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Wed, 21 Sep 2022 23:17:10 +0900
Subject: [PATCH 2991/3199] save

---
 scheduler/src/lib.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 575897da497d29..ed6957301f34ac 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1899,7 +1899,7 @@ impl ScheduleStage {
                     break;
                 } else {
                     if !empty_from_exec {
-                        let mut processed_execution_environment = from_exec.recv().unwrap().0;
+                        let (mut processed_execution_environment, extra) = from_exec.recv().unwrap();
                         from_exec_len = from_exec_len.checked_sub(1).unwrap();
                         empty_from_exec = from_exec_len == 0;
                         executing_queue_count = executing_queue_count.checked_sub(1).unwrap();
@@ -1912,7 +1912,7 @@ impl ScheduleStage {
                             &mut provisioning_tracker_count,
                         );
                         to_next_stage
-                            .send(ExaminablePayload(processed_execution_environment))
+                            .send(ExaminablePayload(processed_execution_environment, extra))
                             .unwrap();
                     }
                     if !empty_from {

From 6f3c1511fc24287e5a69098d4164ae8cb82c5586 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Wed, 21 Sep 2022 23:17:23 +0900
Subject: [PATCH 2992/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index ed6957301f34ac..0d86958aa9efce 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1899,7 +1899,7 @@ impl ScheduleStage {
                     break;
                 } else {
                     if !empty_from_exec {
-                        let (mut processed_execution_environment, extra) = from_exec.recv().unwrap();
+                        let UnlockablePayload(mut processed_execution_environment, extra) = from_exec.recv().unwrap();
                         from_exec_len = from_exec_len.checked_sub(1).unwrap();
                         empty_from_exec = from_exec_len == 0;
                         executing_queue_count = executing_queue_count.checked_sub(1).unwrap();

From 8621c89a42e1040ea0b3c9ac08010ce86706351f Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Wed, 21 Sep 2022 23:17:47 +0900
Subject: [PATCH 2993/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 0d86958aa9efce..a1c893d36aa919 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1987,7 +1987,7 @@ impl ScheduleStage {
         struct AtTopOfScheduleThread;
         unsafe impl AtScheduleThread for AtTopOfScheduleThread {}
 
-        Self::_run::<AtTopOfScheduleThread>(
+        Self::_run::<AtTopOfScheduleThread, T>(
             AtTopOfScheduleThread,
             random_id,
             max_executing_queue_count,

From db0cefdb159d43dce2b05196b08dd2778bf61192 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Wed, 21 Sep 2022 23:18:01 +0900
Subject: [PATCH 2994/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index a1c893d36aa919..7792dac25cd573 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1598,7 +1598,7 @@ impl ScheduleStage {
     }
 
     #[must_use]
-    fn _run<'a, AST: AtScheduleThread, T>(
+    fn _run<'a, AST: AtScheduleThread, T: Send>(
         ast: AST,
         random_id: u64,
         max_executing_queue_count: usize,

From 9ed6c289bf12a584ce0a80bf7f1cacacbc8f72a9 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Wed, 21 Sep 2022 23:18:12 +0900
Subject: [PATCH 2995/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 7792dac25cd573..80f34752df6ebb 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1972,7 +1972,7 @@ impl ScheduleStage {
     }
 
     #[must_use]
-    pub fn run<T>(
+    pub fn run<T: Send>(
         random_id: u64,
         max_executing_queue_count: usize,
         runnable_queue: &mut TaskQueue,

From f8c0f7eeed9c00f33bd0e1baaf8106f520d9486d Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Wed, 21 Sep 2022 23:18:37 +0900
Subject: [PATCH 2996/3199] save

---
 runtime/src/bank.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index aad9a661991942..3a8ae56e20b4fa 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1419,7 +1419,7 @@ impl Scheduler {
                     .unwrap();
                 }
 
-                while let Ok(solana_scheduler::ExaminablePayload(mut ee)) =
+                while let Ok(solana_scheduler::ExaminablePayload(mut ee, _extra)) =
                     retired_ee_receiver.recv()
                 {
                     if send_metrics {

From 978d7dfd90f0a287839f61bd16a18b53dc6ea8d6 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Wed, 21 Sep 2022 23:19:21 +0900
Subject: [PATCH 2997/3199] save

---
 scheduler/src/lib.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 80f34752df6ebb..6df12a613a5220 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -2005,8 +2005,8 @@ impl ScheduleStage {
 
 pub struct SchedulablePayload(pub Flushable<TaskInQueue>);
 pub struct ExecutablePayload(pub Box<ExecutionEnvironment>);
-pub struct UnlockablePayload<T>(pub Box<ExecutionEnvironment>, T);
-pub struct ExaminablePayload<T>(pub Box<ExecutionEnvironment>, T);
+pub struct UnlockablePayload<T>(pub Box<ExecutionEnvironment>, pub T);
+pub struct ExaminablePayload<T>(pub Box<ExecutionEnvironment>, pub T);
 
 pub struct Checkpoint(std::sync::Mutex<usize>, std::sync::Condvar);
 

From 66e78cd7f5fcdd6891fbe4027b258c4426689d71 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Wed, 21 Sep 2022 23:19:49 +0900
Subject: [PATCH 2998/3199] save

---
 ledger-tool/src/main.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ledger-tool/src/main.rs b/ledger-tool/src/main.rs
index a31e8a6bfc1066..4f3c2be5dcdbe2 100644
--- a/ledger-tool/src/main.rs
+++ b/ledger-tool/src/main.rs
@@ -351,7 +351,7 @@ fn output_slot(
                         // ee.reindex_with_address_book();
                         todo!("contended_write_task_count");
                         post_execute_env_sender
-                            .send(solana_scheduler::UnlockablePayload(ee))
+                            .send(solana_scheduler::UnlockablePayload(ee, ()))
                             .unwrap();
                     }
                 })

From 0b126ec1459579a613c48eb7ee997029c97e08eb Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Wed, 21 Sep 2022 23:22:19 +0900
Subject: [PATCH 2999/3199] save

---
 runtime/src/bank.rs | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 3a8ae56e20b4fa..8c4db463a25757 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1370,8 +1370,6 @@ impl Scheduler {
                 drop(weak_bank);
                 drop(ro_bank);
 
-                trace!("execute_substage: slot: {} transaction_index: {} timings: {:?}", slot, transaction_index, timings);
-
                 let TransactionResults {
                     fee_collection_results,
                     execution_results,
@@ -1419,7 +1417,7 @@ impl Scheduler {
                     .unwrap();
                 }
 
-                while let Ok(solana_scheduler::ExaminablePayload(mut ee, _extra)) =
+                while let Ok(solana_scheduler::ExaminablePayload(mut ee, timings)) =
                     retired_ee_receiver.recv()
                 {
                     if send_metrics {
@@ -1442,6 +1440,7 @@ impl Scheduler {
                             ("cpu_duration", ee.execution_cpu_us, i64),
                             ("compute_units", ee.cu, i64),
                         );
+                        info!("execute_substage: slot: {} transaction_index: {} timings: {:?}", ee.slot, ee.transaction_index, timings);
                     }
 
                     if ee.is_aborted() {

From e981d55f848b7da248dc2fff6b82e4401ca6c752 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Wed, 21 Sep 2022 23:43:19 +0900
Subject: [PATCH 3000/3199] save

---
 runtime/src/bank.rs | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 8c4db463a25757..29a0b8662f2bf1 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1417,9 +1417,12 @@ impl Scheduler {
                     .unwrap();
                 }
 
+                let mut cumulative_timings = Default::default();
+
                 while let Ok(solana_scheduler::ExaminablePayload(mut ee, timings)) =
                     retired_ee_receiver.recv()
                 {
+                    cumulative_timings.accumulate(&timings);
                     if send_metrics {
                         let sig = ee.task.tx.0.signature().to_string();
 

From 009f8a7d071b074164389c245be04aaadf85f931 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Wed, 21 Sep 2022 23:43:45 +0900
Subject: [PATCH 3001/3199] save

---
 runtime/src/bank.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 29a0b8662f2bf1..4eb8b61473aa96 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1417,7 +1417,7 @@ impl Scheduler {
                     .unwrap();
                 }
 
-                let mut cumulative_timings = Default::default();
+                let mut cumulative_timings = ExecuteTimings::default();
 
                 while let Ok(solana_scheduler::ExaminablePayload(mut ee, timings)) =
                     retired_ee_receiver.recv()

From 591ba00718f577a121a29dde96d880153d220114 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Wed, 21 Sep 2022 23:44:45 +0900
Subject: [PATCH 3002/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 6df12a613a5220..68a43f8a0476e7 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -2006,7 +2006,7 @@ impl ScheduleStage {
 pub struct SchedulablePayload(pub Flushable<TaskInQueue>);
 pub struct ExecutablePayload(pub Box<ExecutionEnvironment>);
 pub struct UnlockablePayload<T>(pub Box<ExecutionEnvironment>, pub T);
-pub struct ExaminablePayload<T>(pub Box<ExecutionEnvironment>, pub T);
+pub struct ExaminablePayload<T>(pub Flushable<(pub Box<ExecutionEnvironment>, pub T)>);
 
 pub struct Checkpoint(std::sync::Mutex<usize>, std::sync::Condvar);
 

From ce346cdb2c2048cc70b13fb8d0b4a592aca05fd6 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Wed, 21 Sep 2022 23:44:57 +0900
Subject: [PATCH 3003/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 68a43f8a0476e7..30788ceb57d96a 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -2006,7 +2006,7 @@ impl ScheduleStage {
 pub struct SchedulablePayload(pub Flushable<TaskInQueue>);
 pub struct ExecutablePayload(pub Box<ExecutionEnvironment>);
 pub struct UnlockablePayload<T>(pub Box<ExecutionEnvironment>, pub T);
-pub struct ExaminablePayload<T>(pub Flushable<(pub Box<ExecutionEnvironment>, pub T)>);
+pub struct ExaminablePayload<T>(pub Flushable<(Box<ExecutionEnvironment>, T)>);
 
 pub struct Checkpoint(std::sync::Mutex<usize>, std::sync::Condvar);
 

From 35c06fa13756315804885b8a954cb02212e7014a Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Wed, 21 Sep 2022 23:45:30 +0900
Subject: [PATCH 3004/3199] save

---
 scheduler/src/lib.rs | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 30788ceb57d96a..339d5dd85c9c7a 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1643,6 +1643,7 @@ impl ScheduleStage {
                         unsafe impl NotAtScheduleThread for NotAtTopOfScheduleThread {}
                         let nast = NotAtTopOfScheduleThread;
 
+                        /*
                         while let Ok(ExaminablePayload(a, _e)) = ee_receiver.recv() {
                             assert!(a.task.lock_attempts_not_mut(nast).is_empty());
                             //assert!(a.task.sequence_time() != usize::max_value());
@@ -1651,6 +1652,7 @@ impl ScheduleStage {
                             //TaskInQueue::get_mut(&mut a.task).unwrap();
                         }
                         assert_eq!(ee_receiver.len(), 0);
+                        */
                         Ok::<(), ()>(())
                     })
                     .unwrap();

From e7c94b5e31c69287f4ebd73197564a344eef4455 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Wed, 21 Sep 2022 23:46:07 +0900
Subject: [PATCH 3005/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 339d5dd85c9c7a..6a40730029f9bd 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1712,7 +1712,7 @@ impl ScheduleStage {
                            executing_queue_count = executing_queue_count.checked_sub(1).unwrap();
                            processed_count = processed_count.checked_add(1).unwrap();
                            Self::commit_processed_execution(ast, &mut processed_execution_environment, address_book, &mut commit_clock, &mut provisioning_tracker_count);
-                           to_next_stage.send(ExaminablePayload(processed_execution_environment, extra)).unwrap();
+                           to_next_stage.send(ExaminablePayload(Flushable::Payload(processed_execution_environment, extra))).unwrap();
                        } else {
                            assert_eq!(from_exec.len(), 0);
                            from_exec_disconnected = true;

From 32bfb60ddf289b24e79b7eb8e2a23d69d8d14d82 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Wed, 21 Sep 2022 23:46:31 +0900
Subject: [PATCH 3006/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 6a40730029f9bd..4c47992d80d8aa 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1712,7 +1712,7 @@ impl ScheduleStage {
                            executing_queue_count = executing_queue_count.checked_sub(1).unwrap();
                            processed_count = processed_count.checked_add(1).unwrap();
                            Self::commit_processed_execution(ast, &mut processed_execution_environment, address_book, &mut commit_clock, &mut provisioning_tracker_count);
-                           to_next_stage.send(ExaminablePayload(Flushable::Payload(processed_execution_environment, extra))).unwrap();
+                           to_next_stage.send(ExaminablePayload(Flushable::Payload((processed_execution_environment, extra)))).unwrap();
                        } else {
                            assert_eq!(from_exec.len(), 0);
                            from_exec_disconnected = true;

From c4d03e01f7ab96623bedd7b93d494fe777238d9b Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Wed, 21 Sep 2022 23:46:54 +0900
Subject: [PATCH 3007/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 4c47992d80d8aa..375ec350ec7705 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1914,7 +1914,7 @@ impl ScheduleStage {
                             &mut provisioning_tracker_count,
                         );
                         to_next_stage
-                            .send(ExaminablePayload(processed_execution_environment, extra))
+                            .send(ExaminablePayload(Flushable::Payload((processed_execution_environment, extra))))
                             .unwrap();
                     }
                     if !empty_from {

From dff804a0953665fdbfdec064284850675b76240a Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Wed, 21 Sep 2022 23:49:37 +0900
Subject: [PATCH 3008/3199] save

---
 runtime/src/bank.rs | 74 ++++++++++++++++++++++++---------------------
 1 file changed, 39 insertions(+), 35 deletions(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 4eb8b61473aa96..6be227be1b100d 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1419,44 +1419,48 @@ impl Scheduler {
 
                 let mut cumulative_timings = ExecuteTimings::default();
 
-                while let Ok(solana_scheduler::ExaminablePayload(mut ee, timings)) =
-                    retired_ee_receiver.recv()
+                while let Ok(r) = retired_ee_receiver.recv()
                 {
-                    cumulative_timings.accumulate(&timings);
-                    if send_metrics {
-                        let sig = ee.task.tx.0.signature().to_string();
-
-                        datapoint_info_at!(
-                            ee.finish_time.unwrap(),
-                            "individual_tx_stats",
-                            ("slot", ee.slot, i64),
-                            ("index", ee.transaction_index, i64),
-                            ("thread", format!("solScExLane{:02}", ee.thx), String),
-                            ("signature", &sig, String),
-                            ("account_locks_in_json", "{}", String),
-                            (
-                                "status",
-                                format!("{:?}", ee.execution_result.as_ref().unwrap()),
-                                String
-                            ),
-                            ("duration", ee.execution_us, i64),
-                            ("cpu_duration", ee.execution_cpu_us, i64),
-                            ("compute_units", ee.cu, i64),
-                        );
-                        info!("execute_substage: slot: {} transaction_index: {} timings: {:?}", ee.slot, ee.transaction_index, timings);
-                    }
+                    match r {
+                        solana_scheduler::ExaminablePayload(solana_schduler::Flushable::Payload(mut ee, timings)) => {
+                            cumulative_timings.accumulate(&timings);
+                            if send_metrics {
+                                let sig = ee.task.tx.0.signature().to_string();
+
+                                datapoint_info_at!(
+                                    ee.finish_time.unwrap(),
+                                    "individual_tx_stats",
+                                    ("slot", ee.slot, i64),
+                                    ("index", ee.transaction_index, i64),
+                                    ("thread", format!("solScExLane{:02}", ee.thx), String),
+                                    ("signature", &sig, String),
+                                    ("account_locks_in_json", "{}", String),
+                                    (
+                                        "status",
+                                        format!("{:?}", ee.execution_result.as_ref().unwrap()),
+                                        String
+                                    ),
+                                    ("duration", ee.execution_us, i64),
+                                    ("cpu_duration", ee.execution_cpu_us, i64),
+                                    ("compute_units", ee.cu, i64),
+                                );
+                                info!("execute_substage: slot: {} transaction_index: {} timings: {:?}", ee.slot, ee.transaction_index, timings);
+                            }
 
-                    if ee.is_aborted() {
-                        warn!(
-                            "scheduler: Unexpected validator error: {:?}, transaction: {:?}",
-                            ee.execution_result, ee.task.tx.0
-                        );
-                        collected_errors_in_collector_thread
-                            .lock()
-                            .unwrap()
-                            .push(ee.execution_result.take().unwrap());
+                            if ee.is_aborted() {
+                                warn!(
+                                    "scheduler: Unexpected validator error: {:?}, transaction: {:?}",
+                                    ee.execution_result, ee.task.tx.0
+                                );
+                                collected_errors_in_collector_thread
+                                    .lock()
+                                    .unwrap()
+                                    .push(ee.execution_result.take().unwrap());
+                            }
+                            drop(ee);
+                        },
+                        _ => { todo!() },
                     }
-                    drop(ee);
                 }
                 todo!();
 

From afcbffa92770ac3bc8120e7eb5c181dee15fe809 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Wed, 21 Sep 2022 23:50:54 +0900
Subject: [PATCH 3009/3199] save

---
 runtime/src/bank.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 6be227be1b100d..23547b2d19ba0b 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1422,7 +1422,7 @@ impl Scheduler {
                 while let Ok(r) = retired_ee_receiver.recv()
                 {
                     match r {
-                        solana_scheduler::ExaminablePayload(solana_schduler::Flushable::Payload(mut ee, timings)) => {
+                        solana_scheduler::ExaminablePayload(solana_scheduler::Flushable::Payload(mut ee, timings)) => {
                             cumulative_timings.accumulate(&timings);
                             if send_metrics {
                                 let sig = ee.task.tx.0.signature().to_string();

From a58a4c3b2f2f81b2e074edf23b035849317d87ac Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Wed, 21 Sep 2022 23:51:10 +0900
Subject: [PATCH 3010/3199] save

---
 runtime/src/bank.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 23547b2d19ba0b..907e9096a690a6 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1422,7 +1422,7 @@ impl Scheduler {
                 while let Ok(r) = retired_ee_receiver.recv()
                 {
                     match r {
-                        solana_scheduler::ExaminablePayload(solana_scheduler::Flushable::Payload(mut ee, timings)) => {
+                        solana_scheduler::ExaminablePayload(solana_scheduler::Flushable::Payload((mut ee, timings))) => {
                             cumulative_timings.accumulate(&timings);
                             if send_metrics {
                                 let sig = ee.task.tx.0.signature().to_string();

From 9aac063146fa93e6c59eafc33d26bfa381ca6d19 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Wed, 21 Sep 2022 23:52:07 +0900
Subject: [PATCH 3011/3199] save

---
 ledger-tool/src/main.rs | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/ledger-tool/src/main.rs b/ledger-tool/src/main.rs
index 4f3c2be5dcdbe2..1b62e26d1b488e 100644
--- a/ledger-tool/src/main.rs
+++ b/ledger-tool/src/main.rs
@@ -377,6 +377,7 @@ fn output_slot(
             let t3 = std::thread::Builder::new()
                 .name(format!("sol-consumer{}", thx))
                 .spawn(move || loop {
+                    /*
                     let ee = post_schedule_env_receiver.recv().unwrap().0;
                     d.fetch_sub(1, Ordering::Relaxed);
                     let step = step.fetch_add(1, Ordering::Relaxed);
@@ -388,6 +389,8 @@ fn output_slot(
                     if step % 1966 == 0 {
                         error!("finished!: {} {}", step, post_schedule_env_receiver.len());
                     }
+                    */
+                    todo!();
                 })
                 .unwrap();
             t3

From 0d79fce14bab532e307ccf56cd44b5993cf4ce62 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Wed, 21 Sep 2022 23:52:48 +0900
Subject: [PATCH 3012/3199] save

---
 runtime/src/bank.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 907e9096a690a6..df40ecba813ed2 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1459,7 +1459,7 @@ impl Scheduler {
                             }
                             drop(ee);
                         },
-                        _ => { todo!() },
+                        solana_scheduler::ExaminablePayload(solana_scheduler::Flushable::Flush(checkpoint) => { todo!() },
                     }
                 }
                 todo!();

From ee226d2245abe497e3ea2d21d3e903ba338ebcaa Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Wed, 21 Sep 2022 23:53:02 +0900
Subject: [PATCH 3013/3199] save

---
 runtime/src/bank.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index df40ecba813ed2..c715f9842d0c96 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1459,7 +1459,7 @@ impl Scheduler {
                             }
                             drop(ee);
                         },
-                        solana_scheduler::ExaminablePayload(solana_scheduler::Flushable::Flush(checkpoint) => { todo!() },
+                        solana_scheduler::ExaminablePayload(solana_scheduler::Flushable::Flush(checkpoint)) => { todo!() },
                     }
                 }
                 todo!();

From f6f14b857c87ebba0235da229f2fc9939a06d56b Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Wed, 21 Sep 2022 23:55:03 +0900
Subject: [PATCH 3014/3199] save

---
 scheduler/src/lib.rs | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 375ec350ec7705..d19ccbb49df478 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1947,6 +1947,9 @@ impl ScheduleStage {
             }
             assert!(!select_skipped || executing_queue_count > 0);
         }
+        if let Some(checkpoint) = maybe_checkpoint {
+            to_next_stage.send(ExaminablePayload(Flushable::Flush(checkpoint))).unwrap();
+        }
         drop(to_next_stage);
         drop(ee_sender);
         drop(task_sender);

From 4afb2f9f00e87258e924b876306bfdc0d0ba458c Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Wed, 21 Sep 2022 23:55:14 +0900
Subject: [PATCH 3015/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index d19ccbb49df478..3a617f9e562f92 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1947,7 +1947,7 @@ impl ScheduleStage {
             }
             assert!(!select_skipped || executing_queue_count > 0);
         }
-        if let Some(checkpoint) = maybe_checkpoint {
+        if let Some(checkpoint) = &maybe_checkpoint {
             to_next_stage.send(ExaminablePayload(Flushable::Flush(checkpoint))).unwrap();
         }
         drop(to_next_stage);

From 7fbbc0b2d6aa82b1043f0a79edcb3bf519c52ec4 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Wed, 21 Sep 2022 23:55:30 +0900
Subject: [PATCH 3016/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 3a617f9e562f92..e7daa87b47e11d 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1948,7 +1948,7 @@ impl ScheduleStage {
             assert!(!select_skipped || executing_queue_count > 0);
         }
         if let Some(checkpoint) = &maybe_checkpoint {
-            to_next_stage.send(ExaminablePayload(Flushable::Flush(checkpoint))).unwrap();
+            to_next_stage.send(ExaminablePayload(Flushable::Flush(std::sync::Arc::clone(checkpoint)))).unwrap();
         }
         drop(to_next_stage);
         drop(ee_sender);

From dfd4d131508a7000b938b8aa6e5c331a18e6fff3 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Wed, 21 Sep 2022 23:56:33 +0900
Subject: [PATCH 3017/3199] save

---
 runtime/src/bank.rs | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index c715f9842d0c96..fe5e8c4f0ecb37 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1459,7 +1459,10 @@ impl Scheduler {
                             }
                             drop(ee);
                         },
-                        solana_scheduler::ExaminablePayload(solana_scheduler::Flushable::Flush(checkpoint)) => { todo!() },
+                        solana_scheduler::ExaminablePayload(solana_scheduler::Flushable::Flush(checkpoint)) => {
+                            checkpoint.wait_for_restart();
+                            cumulative_timings = ExecuteTimings::default();
+                        },
                     }
                 }
                 todo!();

From 41186fc8025d6eae51033dae18f57ddb393b1aa0 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 22 Sep 2022 00:31:27 +0900
Subject: [PATCH 3018/3199] save

---
 scheduler/src/lib.rs | 30 +++++++++++++++++++-----------
 1 file changed, 19 insertions(+), 11 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index e7daa87b47e11d..cecd801ea6d96a 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -2013,23 +2013,29 @@ pub struct ExecutablePayload(pub Box<ExecutionEnvironment>);
 pub struct UnlockablePayload<T>(pub Box<ExecutionEnvironment>, pub T);
 pub struct ExaminablePayload<T>(pub Flushable<(Box<ExecutionEnvironment>, T)>);
 
-pub struct Checkpoint(std::sync::Mutex<usize>, std::sync::Condvar);
+pub struct Checkpoint<T>(std::sync::Mutex<(usize, Option<T>)>, std::sync::Condvar);
 
-impl Checkpoint {
-    pub fn wait_for_restart(&self) {
+impl<T> Checkpoint {
+    pub fn wait_for_restart(&self, maybe_given_restart_value: Option<T>) {
         let current_thread_name = std::thread::current().name().unwrap().to_string();
-        let mut remaining_threads_guard = self.0.lock().unwrap();
+        let (mut self_remaining_threads_guard, mut self_return_value) = self.0.lock().unwrap();
         info!(
             "Checkpoint::wait_for_restart: {} is entering at {} -> {}",
             current_thread_name,
-            *remaining_threads_guard,
-            *remaining_threads_guard - 1
+            *self_remaining_threads_guard,
+            *self_remaining_threads_guard - 1
         );
 
-        *remaining_threads_guard -= 1;
+        *self_remaining_threads_guard -= 1;
 
-        if *remaining_threads_guard == 0 {
-            drop(remaining_threads_guard);
+        if let Some(given_restart_value) = maybe_given_restart_value {
+            assert!(self_return_value.is_none());
+            *self_return_value = Some(given_restart_value);
+        }
+
+        if *self_remaining_threads_guard == 0 {
+            assert!(self_return_value.is_some());
+            drop(self_remaining_threads_guard);
             self.1.notify_all();
             info!(
                 "Checkpoint::wait_for_restart: {} notified all others...",
@@ -2042,7 +2048,7 @@ impl Checkpoint {
             );
             let _ = *self
                 .1
-                .wait_while(remaining_threads_guard, |&mut remaining_threads| {
+                .wait_while(self_remaining_threads_guard, |&mut remaining_threads| {
                     remaining_threads > 0
                 })
                 .unwrap();
@@ -2051,11 +2057,13 @@ impl Checkpoint {
                 current_thread_name
             );
         }
+
+        r
     }
 
     pub fn new(remaining_threads: usize) -> std::sync::Arc<Self> {
         std::sync::Arc::new(Self(
-            std::sync::Mutex::new(remaining_threads),
+            std::sync::Mutex::new((remaining_threads, None)),
             std::sync::Condvar::new(),
         ))
     }

From 222b6a475e0cb3b469f67f4f3d32d8945d075fc6 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 22 Sep 2022 00:33:33 +0900
Subject: [PATCH 3019/3199] save

---
 scheduler/src/lib.rs | 36 +++++++++++++++++-------------------
 1 file changed, 17 insertions(+), 19 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index cecd801ea6d96a..e969add3f2e66b 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -906,13 +906,13 @@ impl TaskQueue {
     }
 }
 
-struct ChannelBackedTaskQueue<'a> {
+struct ChannelBackedTaskQueue<'a, C> {
     channel: &'a crossbeam_channel::Receiver<SchedulablePayload>,
     buffered_task: Option<TaskInQueue>,
-    buffered_flush: Option<std::sync::Arc<Checkpoint>>,
+    buffered_flush: Option<std::sync::Arc<Checkpoint<C>>>,
 }
 
-impl<'a> ChannelBackedTaskQueue<'a> {
+impl<'a, C> ChannelBackedTaskQueue<'a, C> {
     fn new(channel: &'a crossbeam_channel::Receiver<SchedulablePayload>) -> Self {
         Self {channel, buffered_task: None, buffered_flush: None}
     }
@@ -1068,9 +1068,9 @@ impl ScheduleStage {
     }
 
     #[inline(never)]
-    fn select_next_task<'a>(
+    fn select_next_task<'a, C>(
         //runnable_queue: &'a mut TaskQueue,
-        runnable_queue: &'a mut ChannelBackedTaskQueue,
+        runnable_queue: &'a mut ChannelBackedTaskQueue<C>,
         address_book: &mut AddressBook,
         contended_count: &usize,
         task_selection: &mut TaskSelection,
@@ -1163,11 +1163,11 @@ impl ScheduleStage {
     }
 
     #[inline(never)]
-    fn pop_from_queue_then_lock<AST: AtScheduleThread>(
+    fn pop_from_queue_then_lock<AST: AtScheduleThread, C>(
         ast: AST,
         task_sender: &crossbeam_channel::Sender<(TaskInQueue, Vec<LockAttempt>)>,
         //runnable_queue: &mut TaskQueue,
-        runnable_queue: &mut ChannelBackedTaskQueue,
+        runnable_queue: &mut ChannelBackedTaskQueue<C>,
         address_book: &mut AddressBook,
         contended_count: &mut usize,
         prefer_immediate: bool,
@@ -1548,11 +1548,11 @@ impl ScheduleStage {
     }
 
     #[inline(never)]
-    fn schedule_next_execution<AST: AtScheduleThread>(
+    fn schedule_next_execution<AST: AtScheduleThread, C>(
         ast: AST,
         task_sender: &crossbeam_channel::Sender<(TaskInQueue, Vec<LockAttempt>)>,
         //runnable_queue: &mut TaskQueue,
-        runnable_queue: &mut ChannelBackedTaskQueue,
+        runnable_queue: &mut ChannelBackedTaskQueue<C>,
         address_book: &mut AddressBook,
         contended_count: &mut usize,
         prefer_immediate: bool,
@@ -1598,18 +1598,18 @@ impl ScheduleStage {
     }
 
     #[must_use]
-    fn _run<'a, AST: AtScheduleThread, T: Send>(
+    fn _run<'a, AST: AtScheduleThread, T: Send, C>(
         ast: AST,
         random_id: u64,
         max_executing_queue_count: usize,
         _runnable_queue: &mut TaskQueue,
         address_book: &mut AddressBook,
-        mut from_prev: &'a crossbeam_channel::Receiver<SchedulablePayload>,
+        mut from_prev: &'a crossbeam_channel::Receiver<SchedulablePayload<C>>,
         to_execute_substage: &crossbeam_channel::Sender<ExecutablePayload>,
         to_high_execute_substage: Option<&crossbeam_channel::Sender<ExecutablePayload>>,
         from_exec: &crossbeam_channel::Receiver<UnlockablePayload<T>>,
         maybe_to_next_stage: Option<&crossbeam_channel::Sender<ExaminablePayload<T>>>, // assume nonblocking
-        never: &'a crossbeam_channel::Receiver<SchedulablePayload>,
+        never: &'a crossbeam_channel::Receiver<SchedulablePayload<C>>,
     ) -> Option<std::sync::Arc<Checkpoint>> {
         let mut maybe_start_time = None;
         let (mut last_time, mut last_processed_count) = (maybe_start_time.clone(), 0_usize);
@@ -1977,12 +1977,12 @@ impl ScheduleStage {
     }
 
     #[must_use]
-    pub fn run<T: Send>(
+    pub fn run<T: Send, C>(
         random_id: u64,
         max_executing_queue_count: usize,
         runnable_queue: &mut TaskQueue,
         address_book: &mut AddressBook,
-        from: &crossbeam_channel::Receiver<SchedulablePayload>,
+        from: &crossbeam_channel::Receiver<SchedulablePayload<C>>,
         to_execute_substage: &crossbeam_channel::Sender<ExecutablePayload>,
         to_high_execute_substage: Option<&crossbeam_channel::Sender<ExecutablePayload>>,
         from_execute_substage: &crossbeam_channel::Receiver<UnlockablePayload<T>>,
@@ -2008,7 +2008,7 @@ impl ScheduleStage {
     }
 }
 
-pub struct SchedulablePayload(pub Flushable<TaskInQueue>);
+pub struct SchedulablePayload<C>(pub Flushable<TaskInQueue, C>);
 pub struct ExecutablePayload(pub Box<ExecutionEnvironment>);
 pub struct UnlockablePayload<T>(pub Box<ExecutionEnvironment>, pub T);
 pub struct ExaminablePayload<T>(pub Flushable<(Box<ExecutionEnvironment>, T)>);
@@ -2057,8 +2057,6 @@ impl<T> Checkpoint {
                 current_thread_name
             );
         }
-
-        r
     }
 
     pub fn new(remaining_threads: usize) -> std::sync::Arc<Self> {
@@ -2069,7 +2067,7 @@ impl<T> Checkpoint {
     }
 }
 
-pub enum Flushable<T> {
+pub enum Flushable<T, C> {
     Payload(T),
-    Flush(std::sync::Arc<Checkpoint>),
+    Flush(std::sync::Arc<Checkpoint<C>>),
 }

From 1601d316a0b29547c10b85c7b51ba779db14829d Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 22 Sep 2022 00:33:49 +0900
Subject: [PATCH 3020/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index e969add3f2e66b..e6c4c047a490bd 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1987,7 +1987,7 @@ impl ScheduleStage {
         to_high_execute_substage: Option<&crossbeam_channel::Sender<ExecutablePayload>>,
         from_execute_substage: &crossbeam_channel::Receiver<UnlockablePayload<T>>,
         maybe_to_next_stage: Option<&crossbeam_channel::Sender<ExaminablePayload<T>>>, // assume nonblocking
-    ) -> Option<std::sync::Arc<Checkpoint>> {
+    ) -> Option<std::sync::Arc<Checkpoint<C>>> {
         #[derive(Clone, Copy, Debug)]
         struct AtTopOfScheduleThread;
         unsafe impl AtScheduleThread for AtTopOfScheduleThread {}

From b9b548a76b69baa230c50d5b6a0468f4fc3290ff Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 22 Sep 2022 00:34:14 +0900
Subject: [PATCH 3021/3199] save

---
 scheduler/src/lib.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index e6c4c047a490bd..ccebdc59c43bd5 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -933,7 +933,7 @@ impl<'a, C> ChannelBackedTaskQueue<'a, C> {
         self.task_count_hint() == 0
     }
 
-    fn take_buffered_flush(&mut self) -> Option<std::sync::Arc<Checkpoint>> {
+    fn take_buffered_flush(&mut self) -> Option<std::sync::Arc<Checkpoint<C>>> {
         self.buffered_flush.take()
     }
 
@@ -1610,7 +1610,7 @@ impl ScheduleStage {
         from_exec: &crossbeam_channel::Receiver<UnlockablePayload<T>>,
         maybe_to_next_stage: Option<&crossbeam_channel::Sender<ExaminablePayload<T>>>, // assume nonblocking
         never: &'a crossbeam_channel::Receiver<SchedulablePayload<C>>,
-    ) -> Option<std::sync::Arc<Checkpoint>> {
+    ) -> Option<std::sync::Arc<Checkpoint<C>>> {
         let mut maybe_start_time = None;
         let (mut last_time, mut last_processed_count) = (maybe_start_time.clone(), 0_usize);
         info!("schedule_once:standby id_{:016x}", random_id);

From dcd3234542006ba14111d6c6352f69638b02cc12 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 22 Sep 2022 00:34:33 +0900
Subject: [PATCH 3022/3199] save

---
 scheduler/src/lib.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index ccebdc59c43bd5..4ea084f2e41c76 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -907,13 +907,13 @@ impl TaskQueue {
 }
 
 struct ChannelBackedTaskQueue<'a, C> {
-    channel: &'a crossbeam_channel::Receiver<SchedulablePayload>,
+    channel: &'a crossbeam_channel::Receiver<SchedulablePayload<C>>,
     buffered_task: Option<TaskInQueue>,
     buffered_flush: Option<std::sync::Arc<Checkpoint<C>>>,
 }
 
 impl<'a, C> ChannelBackedTaskQueue<'a, C> {
-    fn new(channel: &'a crossbeam_channel::Receiver<SchedulablePayload>) -> Self {
+    fn new(channel: &'a crossbeam_channel::Receiver<SchedulablePayload<C>>) -> Self {
         Self {channel, buffered_task: None, buffered_flush: None}
     }
 

From 2e277e031721d81516b6c313779791f7ba361080 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 22 Sep 2022 00:35:06 +0900
Subject: [PATCH 3023/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 4ea084f2e41c76..d4ef0c3b62abd9 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -2011,7 +2011,7 @@ impl ScheduleStage {
 pub struct SchedulablePayload<C>(pub Flushable<TaskInQueue, C>);
 pub struct ExecutablePayload(pub Box<ExecutionEnvironment>);
 pub struct UnlockablePayload<T>(pub Box<ExecutionEnvironment>, pub T);
-pub struct ExaminablePayload<T>(pub Flushable<(Box<ExecutionEnvironment>, T)>);
+pub struct ExaminablePayload<T, C>(pub Flushable<(Box<ExecutionEnvironment>, T), C>);
 
 pub struct Checkpoint<T>(std::sync::Mutex<(usize, Option<T>)>, std::sync::Condvar);
 

From 6f624a98f8e31ae16a28d1463e6280047ea96b53 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 22 Sep 2022 00:35:18 +0900
Subject: [PATCH 3024/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index d4ef0c3b62abd9..8211fbd9d6bd1f 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -2015,7 +2015,7 @@ pub struct ExaminablePayload<T, C>(pub Flushable<(Box<ExecutionEnvironment>, T),
 
 pub struct Checkpoint<T>(std::sync::Mutex<(usize, Option<T>)>, std::sync::Condvar);
 
-impl<T> Checkpoint {
+impl<T> Checkpoint<T> {
     pub fn wait_for_restart(&self, maybe_given_restart_value: Option<T>) {
         let current_thread_name = std::thread::current().name().unwrap().to_string();
         let (mut self_remaining_threads_guard, mut self_return_value) = self.0.lock().unwrap();

From 540ebdb245cc5ce3dbe7fda2eed705ec8d1c20d6 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 22 Sep 2022 00:35:30 +0900
Subject: [PATCH 3025/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 8211fbd9d6bd1f..ef06523c0f5b41 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1608,7 +1608,7 @@ impl ScheduleStage {
         to_execute_substage: &crossbeam_channel::Sender<ExecutablePayload>,
         to_high_execute_substage: Option<&crossbeam_channel::Sender<ExecutablePayload>>,
         from_exec: &crossbeam_channel::Receiver<UnlockablePayload<T>>,
-        maybe_to_next_stage: Option<&crossbeam_channel::Sender<ExaminablePayload<T>>>, // assume nonblocking
+        maybe_to_next_stage: Option<&crossbeam_channel::Sender<ExaminablePayload<T, C>>>, // assume nonblocking
         never: &'a crossbeam_channel::Receiver<SchedulablePayload<C>>,
     ) -> Option<std::sync::Arc<Checkpoint<C>>> {
         let mut maybe_start_time = None;

From d8ec3fdcaef4c5365f8e2352fb59afd003391dc6 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 22 Sep 2022 00:35:39 +0900
Subject: [PATCH 3026/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index ef06523c0f5b41..2355c592c9ff29 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1986,7 +1986,7 @@ impl ScheduleStage {
         to_execute_substage: &crossbeam_channel::Sender<ExecutablePayload>,
         to_high_execute_substage: Option<&crossbeam_channel::Sender<ExecutablePayload>>,
         from_execute_substage: &crossbeam_channel::Receiver<UnlockablePayload<T>>,
-        maybe_to_next_stage: Option<&crossbeam_channel::Sender<ExaminablePayload<T>>>, // assume nonblocking
+        maybe_to_next_stage: Option<&crossbeam_channel::Sender<ExaminablePayload<T, C>>>, // assume nonblocking
     ) -> Option<std::sync::Arc<Checkpoint<C>>> {
         #[derive(Clone, Copy, Debug)]
         struct AtTopOfScheduleThread;

From 9baff7b2ebbcb46c39ebb364fd853777c5016108 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 22 Sep 2022 00:35:53 +0900
Subject: [PATCH 3027/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 2355c592c9ff29..08f57b03c75523 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1992,7 +1992,7 @@ impl ScheduleStage {
         struct AtTopOfScheduleThread;
         unsafe impl AtScheduleThread for AtTopOfScheduleThread {}
 
-        Self::_run::<AtTopOfScheduleThread, T>(
+        Self::_run::<AtTopOfScheduleThread, T, C>(
             AtTopOfScheduleThread,
             random_id,
             max_executing_queue_count,

From 5047211f69818cae05821f82fc674c80e86327f9 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 22 Sep 2022 00:36:09 +0900
Subject: [PATCH 3028/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 08f57b03c75523..3e5ecbcf7791df 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1628,7 +1628,7 @@ impl ScheduleStage {
 
         assert!(max_executing_queue_count > 0);
 
-        let (ee_sender, ee_receiver) = crossbeam_channel::unbounded::<ExaminablePayload<T>>();
+        let (ee_sender, ee_receiver) = crossbeam_channel::unbounded::<ExaminablePayload<T, C>>();
 
         let (to_next_stage, maybe_reaper_thread_handle) =
             if let Some(to_next_stage) = maybe_to_next_stage {

From d07b640da3c6ace2f84db95270dbd1bb92105065 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 22 Sep 2022 00:36:44 +0900
Subject: [PATCH 3029/3199] save

---
 scheduler/src/lib.rs | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 3e5ecbcf7791df..efc2d83d2c2300 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -2018,7 +2018,8 @@ pub struct Checkpoint<T>(std::sync::Mutex<(usize, Option<T>)>, std::sync::Condva
 impl<T> Checkpoint<T> {
     pub fn wait_for_restart(&self, maybe_given_restart_value: Option<T>) {
         let current_thread_name = std::thread::current().name().unwrap().to_string();
-        let (mut self_remaining_threads_guard, mut self_return_value) = self.0.lock().unwrap();
+        let g = self.0.lock().unwrap();
+        let (mut self_remaining_threads_guard, mut self_return_value) = &*g;
         info!(
             "Checkpoint::wait_for_restart: {} is entering at {} -> {}",
             current_thread_name,

From a755d00dc3d574867573e88e7bee0a12e936b6f9 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 22 Sep 2022 00:36:57 +0900
Subject: [PATCH 3030/3199] save

---
 scheduler/src/lib.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index efc2d83d2c2300..a47d9ddaa43c08 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -2023,8 +2023,8 @@ impl<T> Checkpoint<T> {
         info!(
             "Checkpoint::wait_for_restart: {} is entering at {} -> {}",
             current_thread_name,
-            *self_remaining_threads_guard,
-            *self_remaining_threads_guard - 1
+            self_remaining_threads_guard,
+            self_remaining_threads_guard - 1
         );
 
         *self_remaining_threads_guard -= 1;

From a2dfa3bdd81d6f94cf9a9db5130d4db63b18d4b3 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 22 Sep 2022 00:37:06 +0900
Subject: [PATCH 3031/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index a47d9ddaa43c08..d5b02b25885d1d 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -2027,7 +2027,7 @@ impl<T> Checkpoint<T> {
             self_remaining_threads_guard - 1
         );
 
-        *self_remaining_threads_guard -= 1;
+        self_remaining_threads_guard -= 1;
 
         if let Some(given_restart_value) = maybe_given_restart_value {
             assert!(self_return_value.is_none());

From bb3b5e1d8739c8f06c33b3fa673a9d1e9e3819d3 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 22 Sep 2022 00:37:17 +0900
Subject: [PATCH 3032/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index d5b02b25885d1d..644ec1f7b62cb9 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -2031,7 +2031,7 @@ impl<T> Checkpoint<T> {
 
         if let Some(given_restart_value) = maybe_given_restart_value {
             assert!(self_return_value.is_none());
-            *self_return_value = Some(given_restart_value);
+            self_return_value = Some(given_restart_value);
         }
 
         if *self_remaining_threads_guard == 0 {

From ea59faa21d0a980aac291bda7d0c2a9e6de73b49 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 22 Sep 2022 00:37:29 +0900
Subject: [PATCH 3033/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 644ec1f7b62cb9..443bed4512982e 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -2034,7 +2034,7 @@ impl<T> Checkpoint<T> {
             self_return_value = Some(given_restart_value);
         }
 
-        if *self_remaining_threads_guard == 0 {
+        if self_remaining_threads_guard == 0 {
             assert!(self_return_value.is_some());
             drop(self_remaining_threads_guard);
             self.1.notify_all();

From aef2fe5d86e86a6491a8abeacc863d4c60efe2e1 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 22 Sep 2022 00:37:48 +0900
Subject: [PATCH 3034/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 443bed4512982e..0071c3bbaa541f 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -2049,7 +2049,7 @@ impl<T> Checkpoint<T> {
             );
             let _ = *self
                 .1
-                .wait_while(self_remaining_threads_guard, |&mut remaining_threads| {
+                .wait_while(g, |&mut remaining_threads| {
                     remaining_threads > 0
                 })
                 .unwrap();

From c48656d26a989e0d510bd9f5f0de00531213b108 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 22 Sep 2022 00:37:59 +0900
Subject: [PATCH 3035/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 0071c3bbaa541f..7fcf88a2d5e94f 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -2049,7 +2049,7 @@ impl<T> Checkpoint<T> {
             );
             let _ = *self
                 .1
-                .wait_while(g, |&mut remaining_threads| {
+                .wait_while(g, |(&mut remaining_threads, _)| {
                     remaining_threads > 0
                 })
                 .unwrap();

From 4a267851369f92589f6f96c3b185892ff93fbb9a Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 22 Sep 2022 00:38:14 +0900
Subject: [PATCH 3036/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 7fcf88a2d5e94f..c8b941771309af 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -2049,7 +2049,7 @@ impl<T> Checkpoint<T> {
             );
             let _ = *self
                 .1
-                .wait_while(g, |(&mut remaining_threads, _)| {
+                .wait_while(g, |(remaining_threads, _)| {
                     remaining_threads > 0
                 })
                 .unwrap();

From 52f3c838887c1005f184f6ec7d21905813410c14 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 22 Sep 2022 00:38:24 +0900
Subject: [PATCH 3037/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index c8b941771309af..86b07a2d305950 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -2049,7 +2049,7 @@ impl<T> Checkpoint<T> {
             );
             let _ = *self
                 .1
-                .wait_while(g, |(remaining_threads, _)| {
+                .wait_while(g, |&mut (remaining_threads, _)| {
                     remaining_threads > 0
                 })
                 .unwrap();

From 33795276410725f998894f4d0c0ce535288caec1 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 22 Sep 2022 00:38:55 +0900
Subject: [PATCH 3038/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 86b07a2d305950..b4d6e20f299bf9 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -2019,7 +2019,7 @@ impl<T> Checkpoint<T> {
     pub fn wait_for_restart(&self, maybe_given_restart_value: Option<T>) {
         let current_thread_name = std::thread::current().name().unwrap().to_string();
         let g = self.0.lock().unwrap();
-        let (mut self_remaining_threads_guard, mut self_return_value) = &*g;
+        let (mut self_remaining_threads_guard, _) = &*g;
         info!(
             "Checkpoint::wait_for_restart: {} is entering at {} -> {}",
             current_thread_name,

From 66146f275e250c06f54c47b5beef3ea57881568c Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 22 Sep 2022 00:39:19 +0900
Subject: [PATCH 3039/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index b4d6e20f299bf9..56f13f0590d00e 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -2019,7 +2019,7 @@ impl<T> Checkpoint<T> {
     pub fn wait_for_restart(&self, maybe_given_restart_value: Option<T>) {
         let current_thread_name = std::thread::current().name().unwrap().to_string();
         let g = self.0.lock().unwrap();
-        let (mut self_remaining_threads_guard, _) = &*g;
+        let (mut self_remaining_threads_guard, mut self_return_value) = &g;
         info!(
             "Checkpoint::wait_for_restart: {} is entering at {} -> {}",
             current_thread_name,

From bd72987c359b72ab58da7070cf36cce40c61cde3 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 22 Sep 2022 00:39:39 +0900
Subject: [PATCH 3040/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 56f13f0590d00e..dba462df200494 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -2019,7 +2019,7 @@ impl<T> Checkpoint<T> {
     pub fn wait_for_restart(&self, maybe_given_restart_value: Option<T>) {
         let current_thread_name = std::thread::current().name().unwrap().to_string();
         let g = self.0.lock().unwrap();
-        let (mut self_remaining_threads_guard, mut self_return_value) = &g;
+        let (mut self_remaining_threads_guard, mut self_return_value) = *g;
         info!(
             "Checkpoint::wait_for_restart: {} is entering at {} -> {}",
             current_thread_name,

From 44e55f96fdf6d580c760dcc61cd1cc46f5a5b8cf Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 22 Sep 2022 00:39:54 +0900
Subject: [PATCH 3041/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index dba462df200494..afe0d312b1a320 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -2019,7 +2019,7 @@ impl<T> Checkpoint<T> {
     pub fn wait_for_restart(&self, maybe_given_restart_value: Option<T>) {
         let current_thread_name = std::thread::current().name().unwrap().to_string();
         let g = self.0.lock().unwrap();
-        let (mut self_remaining_threads_guard, mut self_return_value) = *g;
+        let (&mut self_remaining_threads_guard, &mut self_return_value) = &*g;
         info!(
             "Checkpoint::wait_for_restart: {} is entering at {} -> {}",
             current_thread_name,

From 7f05a8c2acc1f4da859d9250e5b207e4eed72a26 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 22 Sep 2022 00:40:11 +0900
Subject: [PATCH 3042/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index afe0d312b1a320..8c3f98d7185556 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -2019,7 +2019,7 @@ impl<T> Checkpoint<T> {
     pub fn wait_for_restart(&self, maybe_given_restart_value: Option<T>) {
         let current_thread_name = std::thread::current().name().unwrap().to_string();
         let g = self.0.lock().unwrap();
-        let (&mut self_remaining_threads_guard, &mut self_return_value) = &*g;
+        let (&mut self_remaining_threads_guard, &mut self_return_value) = &mut *g;
         info!(
             "Checkpoint::wait_for_restart: {} is entering at {} -> {}",
             current_thread_name,

From 35e4dd5801544ad6de8a3aee3f3cb3fd5e845505 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 22 Sep 2022 00:40:23 +0900
Subject: [PATCH 3043/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 8c3f98d7185556..f11fad7dcbea07 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -2019,7 +2019,7 @@ impl<T> Checkpoint<T> {
     pub fn wait_for_restart(&self, maybe_given_restart_value: Option<T>) {
         let current_thread_name = std::thread::current().name().unwrap().to_string();
         let g = self.0.lock().unwrap();
-        let (&mut self_remaining_threads_guard, &mut self_return_value) = &mut *g;
+        let (self_remaining_threads_guard, self_return_value) = &mut *g;
         info!(
             "Checkpoint::wait_for_restart: {} is entering at {} -> {}",
             current_thread_name,

From 21dab8b8d84457a6264440c1b07bed1b737d4063 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 22 Sep 2022 00:40:37 +0900
Subject: [PATCH 3044/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index f11fad7dcbea07..e208bff3ddd869 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -2027,7 +2027,7 @@ impl<T> Checkpoint<T> {
             self_remaining_threads_guard - 1
         );
 
-        self_remaining_threads_guard -= 1;
+        *self_remaining_threads_guard -= 1;
 
         if let Some(given_restart_value) = maybe_given_restart_value {
             assert!(self_return_value.is_none());

From 92a9ec4e4b3e4175ead7a1eb7ebd47b566903541 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 22 Sep 2022 00:40:48 +0900
Subject: [PATCH 3045/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index e208bff3ddd869..11dad9affe447f 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -2034,7 +2034,7 @@ impl<T> Checkpoint<T> {
             self_return_value = Some(given_restart_value);
         }
 
-        if self_remaining_threads_guard == 0 {
+        if *self_remaining_threads_guard == 0 {
             assert!(self_return_value.is_some());
             drop(self_remaining_threads_guard);
             self.1.notify_all();

From ae6accb0bb38e71bd3c470c262a728e4ac4369a5 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 22 Sep 2022 00:41:20 +0900
Subject: [PATCH 3046/3199] save

---
 scheduler/src/lib.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 11dad9affe447f..2e73311f23b1e1 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -2023,8 +2023,8 @@ impl<T> Checkpoint<T> {
         info!(
             "Checkpoint::wait_for_restart: {} is entering at {} -> {}",
             current_thread_name,
-            self_remaining_threads_guard,
-            self_remaining_threads_guard - 1
+            *self_remaining_threads_guard,
+            *self_remaining_threads_guard - 1
         );
 
         *self_remaining_threads_guard -= 1;

From f67e3a94fb2fd7999ecfd9f5536d1c124f3283fb Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 22 Sep 2022 00:41:38 +0900
Subject: [PATCH 3047/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 2e73311f23b1e1..54172fb8b980aa 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -2031,7 +2031,7 @@ impl<T> Checkpoint<T> {
 
         if let Some(given_restart_value) = maybe_given_restart_value {
             assert!(self_return_value.is_none());
-            self_return_value = Some(given_restart_value);
+            *self_return_value = Some(given_restart_value);
         }
 
         if *self_remaining_threads_guard == 0 {

From 94916625d6abfb07fefb3718cd073d6f7b7ff808 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 22 Sep 2022 00:41:49 +0900
Subject: [PATCH 3048/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 54172fb8b980aa..ca77bdd000c0ce 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -2018,7 +2018,7 @@ pub struct Checkpoint<T>(std::sync::Mutex<(usize, Option<T>)>, std::sync::Condva
 impl<T> Checkpoint<T> {
     pub fn wait_for_restart(&self, maybe_given_restart_value: Option<T>) {
         let current_thread_name = std::thread::current().name().unwrap().to_string();
-        let g = self.0.lock().unwrap();
+        let mut g = self.0.lock().unwrap();
         let (self_remaining_threads_guard, self_return_value) = &mut *g;
         info!(
             "Checkpoint::wait_for_restart: {} is entering at {} -> {}",

From 1ebeeeaba1bee0d68fd3f4776ae05c99951a6ca2 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 22 Sep 2022 00:42:14 +0900
Subject: [PATCH 3049/3199] save

---
 runtime/src/bank.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index fe5e8c4f0ecb37..6f797477f19c28 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1250,12 +1250,12 @@ static SCHEDULER_POOL: std::sync::Mutex<SchedulerPool> =
     std::sync::Mutex::new(SchedulerPool::new());
 
 #[derive(Debug)]
-struct Scheduler {
+struct Scheduler<C> {
     random_id: u64,
     scheduler_thread_handle: Option<std::thread::JoinHandle<Result<(Duration, Duration)>>>,
     executing_thread_handles: Option<Vec<std::thread::JoinHandle<Result<(Duration, Duration)>>>>,
     error_collector_thread_handle: Option<std::thread::JoinHandle<Result<(Duration, Duration)>>>,
-    transaction_sender: Option<crossbeam_channel::Sender<solana_scheduler::SchedulablePayload>>,
+    transaction_sender: Option<crossbeam_channel::Sender<solana_scheduler::SchedulablePayload<C>>>,
     preloader: Arc<solana_scheduler::Preloader>,
     graceful_stop_initiated: AtomicBool,
     collected_errors: Arc<std::sync::Mutex<Vec<Result<()>>>>,

From 0b1c281b4a72bee2c8371ef8b17045522c486a48 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 22 Sep 2022 00:42:42 +0900
Subject: [PATCH 3050/3199] save

---
 runtime/src/bank.rs | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 6f797477f19c28..ebf8e11660fb0f 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1195,11 +1195,11 @@ impl AbiExample for BuiltinPrograms {
     }
 }
 
-struct SchedulerPool {
-    schedulers: Vec<Arc<Scheduler>>,
+struct SchedulerPool<C> {
+    schedulers: Vec<Arc<Scheduler<C>>>,
 }
 
-impl SchedulerPool {
+impl<C> SchedulerPool<C> {
     const fn new() -> Self {
         Self {
             schedulers: Vec::new(),

From fe54d4e9fc5af02949b0c109ce204648f974c2ec Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 22 Sep 2022 09:14:03 +0900
Subject: [PATCH 3051/3199] save

---
 runtime/src/bank.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index ebf8e11660fb0f..d696631d160851 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1617,7 +1617,7 @@ impl Drop for Scheduler {
     }
 }
 
-impl Drop for SchedulerPool {
+impl<C> Drop for SchedulerPool<C> {
     fn drop(&mut self) {
         let current_thread_name = std::thread::current().name().unwrap().to_string();
         warn!("SchedulerPool::drop() by {}...", current_thread_name);

From 23fa43c2bd2edb96c1460c928b1da5a87c733c2a Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 22 Sep 2022 09:14:27 +0900
Subject: [PATCH 3052/3199] save

---
 runtime/src/bank.rs | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index d696631d160851..bfcbfb8ff379f2 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1207,10 +1207,10 @@ impl<C> SchedulerPool<C> {
     }
 
     fn create(&mut self) {
-        self.schedulers.push(Arc::new(Scheduler::default2()));
+        self.schedulers.push(Arc::new(Scheduler<C>::default2()));
     }
 
-    fn take_from_pool(&mut self) -> Arc<Scheduler> {
+    fn take_from_pool(&mut self) -> Arc<Scheduler<C>> {
         if let Some(scheduler) = self.schedulers.pop() {
             info!(
                 "SchedulerPool: id_{:016x} is taken... len: {} => {}",
@@ -1225,7 +1225,7 @@ impl<C> SchedulerPool<C> {
         }
     }
 
-    fn return_to_pool(&mut self, scheduler: Arc<Scheduler>) {
+    fn return_to_pool(&mut self, scheduler: Arc<Scheduler<C>>) {
         info!(
             "SchedulerPool: id_{:016x} is returned... len: {} => {}",
             scheduler.random_id,

From 485669657b3ee48ad7a6978e3e09f0166ee00c19 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 22 Sep 2022 09:14:37 +0900
Subject: [PATCH 3053/3199] save

---
 runtime/src/bank.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index bfcbfb8ff379f2..5f7f2378f1f65c 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1207,7 +1207,7 @@ impl<C> SchedulerPool<C> {
     }
 
     fn create(&mut self) {
-        self.schedulers.push(Arc::new(Scheduler<C>::default2()));
+        self.schedulers.push(Arc::new(Scheduler::<C>::default2()));
     }
 
     fn take_from_pool(&mut self) -> Arc<Scheduler<C>> {

From 26bf81dbb101d7a5b89f5f5377384dc56100f937 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 22 Sep 2022 09:14:54 +0900
Subject: [PATCH 3054/3199] save

---
 runtime/src/bank.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 5f7f2378f1f65c..777c638d0c394e 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1606,7 +1606,7 @@ impl Scheduler {
     }
 }
 
-impl Drop for Scheduler {
+impl<C> Drop for Scheduler<C> {
     fn drop(&mut self) {
         let current_thread_name = std::thread::current().name().unwrap().to_string();
         warn!("Scheduler::drop() by {}...", current_thread_name);

From 5164a5313b8e3bf8f2c41e2c0b8caadc22420a61 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 22 Sep 2022 09:15:07 +0900
Subject: [PATCH 3055/3199] save

---
 runtime/src/bank.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 777c638d0c394e..3042a45e901f36 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1302,7 +1302,7 @@ impl Scheduler {
     }
 }
 
-impl Scheduler {
+impl<C> Scheduler<C> {
     fn default2() -> Self {
         let start = Instant::now();
         let mut address_book = solana_scheduler::AddressBook::default();

From f9c576b0724a42664e4a48894908cc246bd7dce9 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 22 Sep 2022 09:15:21 +0900
Subject: [PATCH 3056/3199] save

---
 runtime/src/bank.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 3042a45e901f36..de014803ee6f9a 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1263,7 +1263,7 @@ struct Scheduler<C> {
     slot: AtomicU64,
 }
 
-impl Scheduler {
+impl<C> Scheduler<C> {
     fn schedule(&self, sanitized_tx: &SanitizedTransaction, index: usize) {
         trace!("Scheduler::schedule()");
         #[derive(Clone, Copy, Debug)]

From a4d7ad55f031150eb854136112132d7b4ccf1be3 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 22 Sep 2022 09:15:32 +0900
Subject: [PATCH 3057/3199] save

---
 runtime/src/bank.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index de014803ee6f9a..73a6f02884c3bf 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1803,7 +1803,7 @@ pub struct Bank {
 
     pub incremental_snapshot_persistence: Option<BankIncrementalSnapshotPersistence>,
 
-    scheduler2: RwLock<Option<Arc<Scheduler>>>,
+    scheduler2: RwLock<Option<Arc<Scheduler<C>>>>,
 }
 
 struct VoteWithStakeDelegations {

From c61f9c0386053082e37d488803fc28dc3072b856 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 22 Sep 2022 09:16:05 +0900
Subject: [PATCH 3058/3199] save

---
 runtime/src/bank.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 73a6f02884c3bf..bfbcb931cd7cb6 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1246,7 +1246,7 @@ impl<C> SchedulerPool<C> {
     }
 }
 
-static SCHEDULER_POOL: std::sync::Mutex<SchedulerPool> =
+static SCHEDULER_POOL: std::sync::Mutex<SchedulerPool<ExecuteTimings>> =
     std::sync::Mutex::new(SchedulerPool::new());
 
 #[derive(Debug)]

From b43b2f3e638191b76b836f38741a1610325def94 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 22 Sep 2022 09:16:22 +0900
Subject: [PATCH 3059/3199] save

---
 runtime/src/bank.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index bfbcb931cd7cb6..fccda0be9e43c6 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1543,7 +1543,7 @@ impl<C> Scheduler<C> {
     }
 }
 
-impl Scheduler {
+impl<C> Scheduler<C> {
     fn gracefully_stop(&self) -> Result<()> {
         if self
             .graceful_stop_initiated

From 1ea96642fcbc73d2298f2a8a27b6256f04aad71e Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 22 Sep 2022 09:16:57 +0900
Subject: [PATCH 3060/3199] save

---
 runtime/src/bank.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index fccda0be9e43c6..e0f3d8419c1012 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1803,7 +1803,7 @@ pub struct Bank {
 
     pub incremental_snapshot_persistence: Option<BankIncrementalSnapshotPersistence>,
 
-    scheduler2: RwLock<Option<Arc<Scheduler<C>>>>,
+    scheduler2: RwLock<Option<Arc<Scheduler<ExecuteTimings>>>>,
 }
 
 struct VoteWithStakeDelegations {

From 4f4fa75607c6d1795c8eeab5046719b0075d7eeb Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 22 Sep 2022 09:17:22 +0900
Subject: [PATCH 3061/3199] save

---
 runtime/src/bank.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index e0f3d8419c1012..d4426dfb1efd23 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1504,7 +1504,7 @@ impl<C> Scheduler<C> {
                     );
 
                     if let Some(checkpoint) = maybe_checkpoint {
-                        checkpoint.wait_for_restart();
+                        checkpoint.wait_for_restart(None);
                         continue;
                     } else {
                         break;
@@ -1573,7 +1573,7 @@ impl<C> Scheduler<C> {
                 solana_scheduler::Flushable::Flush(std::sync::Arc::clone(&checkpoint)),
             ))
             .unwrap();
-        checkpoint.wait_for_restart();
+        checkpoint.wait_for_restart(None);
         {
             *self.bank.write().unwrap() = None;
             self.slot.store(0, std::sync::atomic::Ordering::SeqCst);

From ebd423e2407e58c06f25199dfc1877b117cf9d1c Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 22 Sep 2022 09:17:54 +0900
Subject: [PATCH 3062/3199] save

---
 runtime/src/bank.rs | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index d4426dfb1efd23..2e952fd6789bc0 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1460,8 +1460,7 @@ impl<C> Scheduler<C> {
                             drop(ee);
                         },
                         solana_scheduler::ExaminablePayload(solana_scheduler::Flushable::Flush(checkpoint)) => {
-                            checkpoint.wait_for_restart();
-                            cumulative_timings = ExecuteTimings::default();
+                            checkpoint.wait_for_restart(std::mem::take(&mut cumulative_timings));
                         },
                     }
                 }

From 5c8b8b1acb6e0b58341c8ccf2f0b381c005d9852 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 22 Sep 2022 09:18:10 +0900
Subject: [PATCH 3063/3199] save

---
 runtime/src/bank.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 2e952fd6789bc0..779a3e8cabcb26 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1460,7 +1460,7 @@ impl<C> Scheduler<C> {
                             drop(ee);
                         },
                         solana_scheduler::ExaminablePayload(solana_scheduler::Flushable::Flush(checkpoint)) => {
-                            checkpoint.wait_for_restart(std::mem::take(&mut cumulative_timings));
+                            checkpoint.wait_for_restart(Some(std::mem::take(&mut cumulative_timings)));
                         },
                     }
                 }

From 5d7221816af2587ee582119513691c1e138ab1fc Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 22 Sep 2022 09:19:06 +0900
Subject: [PATCH 3064/3199] save

---
 runtime/src/bank.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 779a3e8cabcb26..26823ee79849fe 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -8379,7 +8379,7 @@ impl Bank {
     }
 
     pub fn wait_for_scheduler(&self, via_drop: bool) -> Result<()> {
-        let s: Option<Arc<Scheduler>> = self.scheduler2.write().unwrap().take();
+        let s: Option<Arc<Scheduler<ExecuteTimings>>> = self.scheduler2.write().unwrap().take();
 
         if let Some(scheduler) = s {
             scheduler.gracefully_stop().unwrap();

From e64b42e2b338d0c2fbee13b94d2f3204e2155614 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 22 Sep 2022 09:20:55 +0900
Subject: [PATCH 3065/3199] save

---
 runtime/src/bank.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 26823ee79849fe..cdd1474bb0b925 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1302,7 +1302,7 @@ impl<C> Scheduler<C> {
     }
 }
 
-impl<C> Scheduler<C> {
+impl Scheduler<ExecuteTimings> {
     fn default2() -> Self {
         let start = Instant::now();
         let mut address_book = solana_scheduler::AddressBook::default();

From 4ed89a43957bdd5cfd3ca381bb85a038cf6bd071 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 22 Sep 2022 09:22:30 +0900
Subject: [PATCH 3066/3199] save

---
 runtime/src/bank.rs | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index cdd1474bb0b925..b99f4f61bc5f49 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1199,7 +1199,7 @@ struct SchedulerPool<C> {
     schedulers: Vec<Arc<Scheduler<C>>>,
 }
 
-impl<C> SchedulerPool<C> {
+impl SchedulerPool<ExecuteTimings> {
     const fn new() -> Self {
         Self {
             schedulers: Vec::new(),
@@ -1207,10 +1207,10 @@ impl<C> SchedulerPool<C> {
     }
 
     fn create(&mut self) {
-        self.schedulers.push(Arc::new(Scheduler::<C>::default2()));
+        self.schedulers.push(Arc::new(Scheduler::<ExecuteTimings>::default2()));
     }
 
-    fn take_from_pool(&mut self) -> Arc<Scheduler<C>> {
+    fn take_from_pool(&mut self) -> Arc<Scheduler<ExecuteTimings>> {
         if let Some(scheduler) = self.schedulers.pop() {
             info!(
                 "SchedulerPool: id_{:016x} is taken... len: {} => {}",
@@ -1225,7 +1225,7 @@ impl<C> SchedulerPool<C> {
         }
     }
 
-    fn return_to_pool(&mut self, scheduler: Arc<Scheduler<C>>) {
+    fn return_to_pool(&mut self, scheduler: Arc<Scheduler<ExecuteTimings>>) {
         info!(
             "SchedulerPool: id_{:016x} is returned... len: {} => {}",
             scheduler.random_id,

From 59daf583fd393bfb5b48b72b78a81e39b087b776 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 22 Sep 2022 09:27:37 +0900
Subject: [PATCH 3067/3199] save

---
 scheduler/src/lib.rs | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index ca77bdd000c0ce..2edb5bdf9e1ef8 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -2060,6 +2060,12 @@ impl<T> Checkpoint<T> {
         }
     }
 
+    pub fn take_restart_value(&self) -> T {
+        let mut g = self.0.lock().unwrap();
+        let (self_remaining_threads_guard, self_return_value) = &mut *g;
+        self_return_value.take()
+    }
+
     pub fn new(remaining_threads: usize) -> std::sync::Arc<Self> {
         std::sync::Arc::new(Self(
             std::sync::Mutex::new((remaining_threads, None)),

From 9c2e21d964fdc59b882638dd0443e5a32b0f6bd1 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 22 Sep 2022 09:27:47 +0900
Subject: [PATCH 3068/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 2edb5bdf9e1ef8..8d54009643e3ef 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -2063,7 +2063,7 @@ impl<T> Checkpoint<T> {
     pub fn take_restart_value(&self) -> T {
         let mut g = self.0.lock().unwrap();
         let (self_remaining_threads_guard, self_return_value) = &mut *g;
-        self_return_value.take()
+        self_return_value.take().unwrap()
     }
 
     pub fn new(remaining_threads: usize) -> std::sync::Arc<Self> {

From fb00242aec4fabcf00f42aa9519aeea1f292488b Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 22 Sep 2022 09:28:12 +0900
Subject: [PATCH 3069/3199] save

---
 runtime/src/bank.rs | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index b99f4f61bc5f49..f5776c33aa64bd 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1564,7 +1564,7 @@ impl<C> Scheduler<C> {
         //let transaction_sender = self.transaction_sender.take().unwrap();
 
         //drop(transaction_sender);
-        let checkpoint = solana_scheduler::Checkpoint::new(2);
+        let checkpoint = solana_scheduler::Checkpoint::new(3);
         self.transaction_sender
             .as_ref()
             .unwrap()
@@ -1573,6 +1573,7 @@ impl<C> Scheduler<C> {
             ))
             .unwrap();
         checkpoint.wait_for_restart(None);
+        let r = checkpoint.take_restart_value();
         {
             *self.bank.write().unwrap() = None;
             self.slot.store(0, std::sync::atomic::Ordering::SeqCst);
@@ -1597,7 +1598,7 @@ impl<C> Scheduler<C> {
         info!("Scheduler::gracefully_stop(): slot: {} id_{:016x} durations 2/2 (wall): scheduler: {}us, error_collector: {}us, lanes: {}us = {:?}", self.slot.map(|s| format!("{}", s)).unwrap_or("-".into()), self.random_id, scheduler_thread_wall_time_us, error_collector_thread_wall_time_us, executing_thread_wall_time_us.iter().sum::<u128>(), &executing_thread_wall_time_us);
         */
 
-        Ok(())
+        Ok(r)
     }
 
     fn handle_aborted_executions(&self) -> Vec<Result<()>> {

From c4996c3e23092b30c738f57f47807691c1cb5496 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 22 Sep 2022 09:28:32 +0900
Subject: [PATCH 3070/3199] save

---
 runtime/src/bank.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index f5776c33aa64bd..462b3f1bf91800 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1543,7 +1543,7 @@ impl Scheduler<ExecuteTimings> {
 }
 
 impl<C> Scheduler<C> {
-    fn gracefully_stop(&self) -> Result<()> {
+    fn gracefully_stop(&self) -> Result<C> {
         if self
             .graceful_stop_initiated
             .load(std::sync::atomic::Ordering::SeqCst)

From c0dfd45c02db7b911a27c6b455ba6f9b205a2279 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 22 Sep 2022 09:30:00 +0900
Subject: [PATCH 3071/3199] save

---
 runtime/src/bank.rs | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 462b3f1bf91800..c6dd8847b14e88 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1543,16 +1543,16 @@ impl Scheduler<ExecuteTimings> {
 }
 
 impl<C> Scheduler<C> {
-    fn gracefully_stop(&self) -> Result<C> {
+    fn gracefully_stop(&self) -> Result<Option<C>> {
         if self
             .graceful_stop_initiated
             .load(std::sync::atomic::Ordering::SeqCst)
         {
-            info!(
+            warn!(
                 "Scheduler::gracefully_stop(): id_{:016x} (skipped..?)",
                 self.random_id
             );
-            return Ok(());
+            return Ok(None);
         }
         self.graceful_stop_initiated
             .store(true, std::sync::atomic::Ordering::SeqCst);

From 500c32365da0f776d6b732a769f5316e069ef315 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 22 Sep 2022 09:30:14 +0900
Subject: [PATCH 3072/3199] save

---
 runtime/src/bank.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index c6dd8847b14e88..40c42c963965b1 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1598,7 +1598,7 @@ impl<C> Scheduler<C> {
         info!("Scheduler::gracefully_stop(): slot: {} id_{:016x} durations 2/2 (wall): scheduler: {}us, error_collector: {}us, lanes: {}us = {:?}", self.slot.map(|s| format!("{}", s)).unwrap_or("-".into()), self.random_id, scheduler_thread_wall_time_us, error_collector_thread_wall_time_us, executing_thread_wall_time_us.iter().sum::<u128>(), &executing_thread_wall_time_us);
         */
 
-        Ok(r)
+        Ok(Some(r))
     }
 
     fn handle_aborted_executions(&self) -> Vec<Result<()>> {

From 679cbffac1f491e962b04ac47b94c9c64593f886 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 22 Sep 2022 09:31:43 +0900
Subject: [PATCH 3073/3199] save

---
 runtime/src/bank.rs | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 40c42c963965b1..1ca8883b7bf274 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -8379,16 +8379,16 @@ impl Bank {
         total_accounts_stats
     }
 
-    pub fn wait_for_scheduler(&self, via_drop: bool) -> Result<()> {
+    pub fn wait_for_scheduler(&self, via_drop: bool) -> Result<usize> {
         let s: Option<Arc<Scheduler<ExecuteTimings>>> = self.scheduler2.write().unwrap().take();
 
         if let Some(scheduler) = s {
-            scheduler.gracefully_stop().unwrap();
+            let r = scheduler.gracefully_stop().unwrap();
             let e = scheduler
                 .handle_aborted_executions()
                 .into_iter()
                 .next()
-                .unwrap_or(Ok(()));
+                .unwrap_or(Ok(r));
             SCHEDULER_POOL.lock().unwrap().return_to_pool(scheduler);
             e
         } else {
@@ -8398,7 +8398,7 @@ impl Bank {
                 via_drop, current_thread_name
             );
 
-            Ok(())
+            Ok(None)
         }
     }
 }

From ddf31524920060c3c621f5a7f64690ddda3d8c68 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 22 Sep 2022 09:32:45 +0900
Subject: [PATCH 3074/3199] save

---
 runtime/src/bank.rs | 1 +
 1 file changed, 1 insertion(+)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 1ca8883b7bf274..1b70ceea4b3307 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -8388,6 +8388,7 @@ impl Bank {
                 .handle_aborted_executions()
                 .into_iter()
                 .next()
+                .map(|_| r)
                 .unwrap_or(Ok(r));
             SCHEDULER_POOL.lock().unwrap().return_to_pool(scheduler);
             e

From a7b85f215e8e6ec28a4991064f6d5d505df22482 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 22 Sep 2022 09:33:07 +0900
Subject: [PATCH 3075/3199] save

---
 runtime/src/bank.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 1b70ceea4b3307..345a94e2bcc49a 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -8379,7 +8379,7 @@ impl Bank {
         total_accounts_stats
     }
 
-    pub fn wait_for_scheduler(&self, via_drop: bool) -> Result<usize> {
+    pub fn wait_for_scheduler(&self, via_drop: bool) -> Result<ExecuteTimings> {
         let s: Option<Arc<Scheduler<ExecuteTimings>>> = self.scheduler2.write().unwrap().take();
 
         if let Some(scheduler) = s {

From f40e51ba8b74b5a4c12904d949981efe968a48fe Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 22 Sep 2022 09:33:22 +0900
Subject: [PATCH 3076/3199] save

---
 runtime/src/bank.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 345a94e2bcc49a..0ca6845005106e 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -8379,7 +8379,7 @@ impl Bank {
         total_accounts_stats
     }
 
-    pub fn wait_for_scheduler(&self, via_drop: bool) -> Result<ExecuteTimings> {
+    pub fn wait_for_scheduler(&self, via_drop: bool) -> Result<Option<ExecuteTimings>> {
         let s: Option<Arc<Scheduler<ExecuteTimings>>> = self.scheduler2.write().unwrap().take();
 
         if let Some(scheduler) = s {

From 0ebfc1c44da273fb0cbc157ee650220cdc765b2c Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 22 Sep 2022 09:34:01 +0900
Subject: [PATCH 3077/3199] save

---
 runtime/src/bank.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 0ca6845005106e..987f4c22345442 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -8389,7 +8389,7 @@ impl Bank {
                 .into_iter()
                 .next()
                 .map(|_| r)
-                .unwrap_or(Ok(r));
+                .or(Ok(r));
             SCHEDULER_POOL.lock().unwrap().return_to_pool(scheduler);
             e
         } else {

From bbdc6943fe5c5a03961bc357eaf3b6b25d1a9284 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 22 Sep 2022 09:34:53 +0900
Subject: [PATCH 3078/3199] save

---
 runtime/src/bank.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 987f4c22345442..32bd31bc991e6f 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -8388,7 +8388,7 @@ impl Bank {
                 .handle_aborted_executions()
                 .into_iter()
                 .next()
-                .map(|_| r)
+                .map_ok(|_| r)
                 .or(Ok(r));
             SCHEDULER_POOL.lock().unwrap().return_to_pool(scheduler);
             e

From 6dd676b01a3c24c32e86d8d54c0c6fbcef2e4eb4 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 22 Sep 2022 09:35:41 +0900
Subject: [PATCH 3079/3199] save

---
 runtime/src/bank.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 32bd31bc991e6f..2348b435b3ce52 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -4315,7 +4315,7 @@ impl Bank {
                 .collected_errors
                 .lock()
                 .unwrap()
-                .push(maybe_last_error);
+                .push(maybe_last_error.map(|_| ()));
         }
         //*self.scheduler.write().unwrap() = new_scheduler;
 

From 428e1d27d61adf84e9939451321114e01fb39b88 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 22 Sep 2022 09:36:31 +0900
Subject: [PATCH 3080/3199] save

---
 runtime/src/bank.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 2348b435b3ce52..18cf744b4002ad 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1258,7 +1258,7 @@ struct Scheduler<C> {
     transaction_sender: Option<crossbeam_channel::Sender<solana_scheduler::SchedulablePayload<C>>>,
     preloader: Arc<solana_scheduler::Preloader>,
     graceful_stop_initiated: AtomicBool,
-    collected_errors: Arc<std::sync::Mutex<Vec<Result<()>>>>,
+    collected_errors: Arc<std::sync::Mutex<Vec<Result<C>>>>,
     bank: std::sync::Arc<std::sync::RwLock<std::option::Option<std::sync::Weak<Bank>>>>,
     slot: AtomicU64,
 }

From a1b3fe252dd08cb59a1e497665328b2fbdbfdcfa Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 22 Sep 2022 09:37:07 +0900
Subject: [PATCH 3081/3199] save

---
 runtime/src/bank.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 18cf744b4002ad..09f855c234e246 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1601,7 +1601,7 @@ impl<C> Scheduler<C> {
         Ok(Some(r))
     }
 
-    fn handle_aborted_executions(&self) -> Vec<Result<()>> {
+    fn handle_aborted_executions(&self) -> Vec<Result<C>> {
         std::mem::take(&mut self.collected_errors.lock().unwrap())
     }
 }

From a4986a3451d565bae0b2ee9108e3a0038660c528 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 22 Sep 2022 09:38:16 +0900
Subject: [PATCH 3082/3199] save

---
 runtime/src/bank.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 09f855c234e246..f205eab020b7d8 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -6723,7 +6723,7 @@ impl Bank {
         }
     }
 
-    pub fn handle_aborted_transactions(&self) -> Vec<Result<()>> {
+    pub fn handle_aborted_transactions(&self) -> Vec<Result<ExecuteTimings>> {
         let s = self.scheduler2.read().unwrap();
         let scheduler = s.as_ref().unwrap();
         scheduler.handle_aborted_executions()

From 8b557e2698a68e0b2feb83d3898e59004385a9a3 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 22 Sep 2022 09:38:49 +0900
Subject: [PATCH 3083/3199] save

---
 runtime/src/bank.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index f205eab020b7d8..07d7813c518b8e 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -8387,8 +8387,8 @@ impl Bank {
             let e = scheduler
                 .handle_aborted_executions()
                 .into_iter()
-                .next()
                 .map_ok(|_| r)
+                .next()
                 .or(Ok(r));
             SCHEDULER_POOL.lock().unwrap().return_to_pool(scheduler);
             e

From 9fdae55ce55db69c78207a06136350cd668e57b7 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 22 Sep 2022 09:39:03 +0900
Subject: [PATCH 3084/3199] save

---
 runtime/src/bank.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 07d7813c518b8e..dab035bc3a9913 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -8389,7 +8389,7 @@ impl Bank {
                 .into_iter()
                 .map_ok(|_| r)
                 .next()
-                .or(Ok(r));
+                .or(r);
             SCHEDULER_POOL.lock().unwrap().return_to_pool(scheduler);
             e
         } else {

From 1bb8e906de5adc26f3a1d5b38e4670ad70fbc513 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 22 Sep 2022 09:40:45 +0900
Subject: [PATCH 3085/3199] save

---
 runtime/src/bank.rs | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index dab035bc3a9913..117b24c94a9e2c 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -8383,13 +8383,10 @@ impl Bank {
         let s: Option<Arc<Scheduler<ExecuteTimings>>> = self.scheduler2.write().unwrap().take();
 
         if let Some(scheduler) = s {
-            let r = scheduler.gracefully_stop().unwrap();
             let e = scheduler
                 .handle_aborted_executions()
                 .into_iter()
-                .map_ok(|_| r)
-                .next()
-                .or(r);
+                .next().unwrap();
             SCHEDULER_POOL.lock().unwrap().return_to_pool(scheduler);
             e
         } else {

From b2f19da1e4a2a56732d887a0b89d01a2c4df22af Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 22 Sep 2022 09:40:54 +0900
Subject: [PATCH 3086/3199] save

---
 runtime/src/bank.rs | 1 +
 1 file changed, 1 insertion(+)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 117b24c94a9e2c..3807762216efd0 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -8383,6 +8383,7 @@ impl Bank {
         let s: Option<Arc<Scheduler<ExecuteTimings>>> = self.scheduler2.write().unwrap().take();
 
         if let Some(scheduler) = s {
+            let _r = scheduler.gracefully_stop().unwrap();
             let e = scheduler
                 .handle_aborted_executions()
                 .into_iter()

From 681a46ecc0d87666e80127468a37cfbe530138be Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 22 Sep 2022 09:41:38 +0900
Subject: [PATCH 3087/3199] save

---
 runtime/src/bank.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 3807762216efd0..f46f14c5df949c 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1455,7 +1455,7 @@ impl Scheduler<ExecuteTimings> {
                                 collected_errors_in_collector_thread
                                     .lock()
                                     .unwrap()
-                                    .push(ee.execution_result.take().unwrap());
+                                    .push(ee.execution_result.take().unwrap().map(|a| Default::default()));
                             }
                             drop(ee);
                         },

From 2633fcb2aff5c83b67542bf5f5a4b6f26dd4975c Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 22 Sep 2022 09:42:18 +0900
Subject: [PATCH 3088/3199] save

---
 runtime/src/bank.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index f46f14c5df949c..fecd2b396d3f83 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -4315,7 +4315,7 @@ impl Bank {
                 .collected_errors
                 .lock()
                 .unwrap()
-                .push(maybe_last_error.map(|_| ()));
+                .push(maybe_last_error);
         }
         //*self.scheduler.write().unwrap() = new_scheduler;
 

From 85f31b70cdf114319f485ba24eddd2b923cf064f Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 22 Sep 2022 09:42:50 +0900
Subject: [PATCH 3089/3199] save

---
 runtime/src/bank.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index fecd2b396d3f83..c7e48c6205b281 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1258,7 +1258,7 @@ struct Scheduler<C> {
     transaction_sender: Option<crossbeam_channel::Sender<solana_scheduler::SchedulablePayload<C>>>,
     preloader: Arc<solana_scheduler::Preloader>,
     graceful_stop_initiated: AtomicBool,
-    collected_errors: Arc<std::sync::Mutex<Vec<Result<C>>>>,
+    collected_errors: Arc<std::sync::Mutex<Vec<Result<Option<C>>>>>,
     bank: std::sync::Arc<std::sync::RwLock<std::option::Option<std::sync::Weak<Bank>>>>,
     slot: AtomicU64,
 }

From 5403590b4423c114c3a6b2b70c8a4bcf12461eff Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 22 Sep 2022 09:43:17 +0900
Subject: [PATCH 3090/3199] save

---
 runtime/src/bank.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index c7e48c6205b281..0c7ffe1248bc41 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1601,7 +1601,7 @@ impl<C> Scheduler<C> {
         Ok(Some(r))
     }
 
-    fn handle_aborted_executions(&self) -> Vec<Result<C>> {
+    fn handle_aborted_executions(&self) -> Vec<Result<Option<C>>> {
         std::mem::take(&mut self.collected_errors.lock().unwrap())
     }
 }

From d7099391a0aed27ee0b2f087b65146de19b573f0 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 22 Sep 2022 09:43:37 +0900
Subject: [PATCH 3091/3199] save

---
 runtime/src/bank.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 0c7ffe1248bc41..e295d12978d7d3 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -6723,7 +6723,7 @@ impl Bank {
         }
     }
 
-    pub fn handle_aborted_transactions(&self) -> Vec<Result<ExecuteTimings>> {
+    pub fn handle_aborted_transactions(&self) -> Vec<Result<Option<ExecuteTimings>>> {
         let s = self.scheduler2.read().unwrap();
         let scheduler = s.as_ref().unwrap();
         scheduler.handle_aborted_executions()

From 778cbeabb6cb98aaa5ad1565a802634ee36e3f1b Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 22 Sep 2022 09:45:10 +0900
Subject: [PATCH 3092/3199] save

---
 core/src/replay_stage.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/core/src/replay_stage.rs b/core/src/replay_stage.rs
index e2c9dc46d6df2f..bb7536cefb7a3f 100644
--- a/core/src/replay_stage.rs
+++ b/core/src/replay_stage.rs
@@ -2477,7 +2477,7 @@ impl ReplayStage {
             assert_eq!(bank_slot, bank.slot());
             if bank.is_complete() {
                 info!("waiting for completed bank: slot: {}", bank.slot());
-                let r = bank.wait_for_scheduler(false);
+                let r: usize = bank.wait_for_scheduler(false);
                 if let Err(err) = r {
                     // Error means the slot needs to be marked as dead
                     Self::mark_dead_slot(

From a204f26a26c3e8cd4949a776e6d84bfb8fab954a Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 22 Sep 2022 09:45:28 +0900
Subject: [PATCH 3093/3199] save

---
 core/src/replay_stage.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/core/src/replay_stage.rs b/core/src/replay_stage.rs
index bb7536cefb7a3f..6401f1a919a0a5 100644
--- a/core/src/replay_stage.rs
+++ b/core/src/replay_stage.rs
@@ -2477,7 +2477,7 @@ impl ReplayStage {
             assert_eq!(bank_slot, bank.slot());
             if bank.is_complete() {
                 info!("waiting for completed bank: slot: {}", bank.slot());
-                let r: usize = bank.wait_for_scheduler(false);
+                let r: usize = bank.wait_for_scheduler(false).unwrap();
                 if let Err(err) = r {
                     // Error means the slot needs to be marked as dead
                     Self::mark_dead_slot(

From 5ad2b6893da2dc460c087aa5ddba9b86a3f104fd Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 22 Sep 2022 09:50:11 +0900
Subject: [PATCH 3094/3199] save

---
 runtime/src/bank.rs | 18 +++++++++---------
 1 file changed, 9 insertions(+), 9 deletions(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index e295d12978d7d3..89408289e48329 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1233,7 +1233,7 @@ impl SchedulerPool<ExecuteTimings> {
             self.schedulers.len() + 1
         );
         assert_eq!(1, Arc::strong_count(&scheduler));
-        assert!(scheduler.collected_errors.lock().unwrap().is_empty());
+        assert!(scheduler.collected_results.lock().unwrap().is_empty());
         assert!(scheduler
             .graceful_stop_initiated
             .load(std::sync::atomic::Ordering::SeqCst));
@@ -1258,7 +1258,7 @@ struct Scheduler<C> {
     transaction_sender: Option<crossbeam_channel::Sender<solana_scheduler::SchedulablePayload<C>>>,
     preloader: Arc<solana_scheduler::Preloader>,
     graceful_stop_initiated: AtomicBool,
-    collected_errors: Arc<std::sync::Mutex<Vec<Result<Option<C>>>>>,
+    collected_results: Arc<std::sync::Mutex<Vec<Result<Option<C>>>>>,
     bank: std::sync::Arc<std::sync::RwLock<std::option::Option<std::sync::Weak<Bank>>>>,
     slot: AtomicU64,
 }
@@ -1403,8 +1403,8 @@ impl Scheduler<ExecuteTimings> {
             Ok((started.0.elapsed(), started.1.elapsed()))
         }).unwrap()}).collect();
 
-        let collected_errors = Arc::new(std::sync::Mutex::new(Vec::new()));
-        let collected_errors_in_collector_thread = Arc::clone(&collected_errors);
+        let collected_results = Arc::new(std::sync::Mutex::new(Vec::new()));
+        let collected_results_in_collector_thread = Arc::clone(&collected_results);
 
         let error_collector_thread_handle = std::thread::Builder::new()
             .name(format!("solScErrCol{:02}", 0))
@@ -1452,7 +1452,7 @@ impl Scheduler<ExecuteTimings> {
                                     "scheduler: Unexpected validator error: {:?}, transaction: {:?}",
                                     ee.execution_result, ee.task.tx.0
                                 );
-                                collected_errors_in_collector_thread
+                                collected_results_in_collector_thread
                                     .lock()
                                     .unwrap()
                                     .push(ee.execution_result.take().unwrap().map(|a| Default::default()));
@@ -1528,7 +1528,7 @@ impl Scheduler<ExecuteTimings> {
             transaction_sender: Some(transaction_sender),
             preloader,
             graceful_stop_initiated: Default::default(),
-            collected_errors,
+            collected_results,
             bank,
             slot: Default::default(),
         };
@@ -1602,7 +1602,7 @@ impl<C> Scheduler<C> {
     }
 
     fn handle_aborted_executions(&self) -> Vec<Result<Option<C>>> {
-        std::mem::take(&mut self.collected_errors.lock().unwrap())
+        std::mem::take(&mut self.collected_results.lock().unwrap())
     }
 }
 
@@ -4309,10 +4309,10 @@ impl Bank {
                 "register_recent_blockhash: carrying over this error: {:?}",
                 maybe_last_error
             );
-            //new_scheduler.collected_errors.lock().unwrap().push(maybe_last_error);
+            //new_scheduler.collected_results.lock().unwrap().push(maybe_last_error);
             s2.as_ref()
                 .unwrap()
-                .collected_errors
+                .collected_results
                 .lock()
                 .unwrap()
                 .push(maybe_last_error);

From 4a40bf1756bfc874544bdd23e8f80070cd5e23bc Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 22 Sep 2022 09:52:13 +0900
Subject: [PATCH 3095/3199] save

---
 runtime/src/bank.rs | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 89408289e48329..9917b8e9877f9c 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -4294,7 +4294,7 @@ impl Bank {
             self.slot()
         );
 
-        let maybe_last_error = self.wait_for_scheduler(false);
+        let last_result = self.wait_for_scheduler(false);
         let scheduler = SCHEDULER_POOL.lock().unwrap().take_from_pool();
         let mut s2 = self.scheduler2.write().unwrap();
         *s2 = Some(scheduler);
@@ -4304,19 +4304,19 @@ impl Bank {
         // much if the write lock is acquired for each tick.
         let mut w_blockhash_queue = self.blockhash_queue.write().unwrap();
         //let new_scheduler = Scheduler::default();
-        if maybe_last_error.is_err() {
+        if last_error.is_err() {
             warn!(
                 "register_recent_blockhash: carrying over this error: {:?}",
                 maybe_last_error
             );
+        }
             //new_scheduler.collected_results.lock().unwrap().push(maybe_last_error);
             s2.as_ref()
                 .unwrap()
                 .collected_results
                 .lock()
                 .unwrap()
-                .push(maybe_last_error);
-        }
+                .push(last_result);
         //*self.scheduler.write().unwrap() = new_scheduler;
 
         info!(

From 35674e8ea26bd5624e3d08d2a3d6f95798a516c5 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 22 Sep 2022 09:52:37 +0900
Subject: [PATCH 3096/3199] save

---
 runtime/src/bank.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 9917b8e9877f9c..fdc5c59adbe631 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -4304,7 +4304,7 @@ impl Bank {
         // much if the write lock is acquired for each tick.
         let mut w_blockhash_queue = self.blockhash_queue.write().unwrap();
         //let new_scheduler = Scheduler::default();
-        if last_error.is_err() {
+        if last_result.is_err() {
             warn!(
                 "register_recent_blockhash: carrying over this error: {:?}",
                 maybe_last_error

From efd6fcf10e3312971a99d6f472883eff73ba3336 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 22 Sep 2022 09:52:49 +0900
Subject: [PATCH 3097/3199] save

---
 runtime/src/bank.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index fdc5c59adbe631..4a77d3a66fe85a 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -4307,7 +4307,7 @@ impl Bank {
         if last_result.is_err() {
             warn!(
                 "register_recent_blockhash: carrying over this error: {:?}",
-                maybe_last_error
+                last_result
             );
         }
             //new_scheduler.collected_results.lock().unwrap().push(maybe_last_error);

From 471e597bf8e4bb9646dc0fd1b4e7029e4311bb6f Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 22 Sep 2022 09:55:05 +0900
Subject: [PATCH 3098/3199] save

---
 runtime/src/bank.rs | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 4a77d3a66fe85a..9e3b1bdeee356a 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1543,7 +1543,7 @@ impl Scheduler<ExecuteTimings> {
 }
 
 impl<C> Scheduler<C> {
-    fn gracefully_stop(&self) -> Result<Option<C>> {
+    fn gracefully_stop(&self) -> Result<()> {
         if self
             .graceful_stop_initiated
             .load(std::sync::atomic::Ordering::SeqCst)
@@ -1552,7 +1552,7 @@ impl<C> Scheduler<C> {
                 "Scheduler::gracefully_stop(): id_{:016x} (skipped..?)",
                 self.random_id
             );
-            return Ok(None);
+            return Ok(());
         }
         self.graceful_stop_initiated
             .store(true, std::sync::atomic::Ordering::SeqCst);
@@ -1574,6 +1574,7 @@ impl<C> Scheduler<C> {
             .unwrap();
         checkpoint.wait_for_restart(None);
         let r = checkpoint.take_restart_value();
+        self.collected_results.push(r);
         {
             *self.bank.write().unwrap() = None;
             self.slot.store(0, std::sync::atomic::Ordering::SeqCst);
@@ -1598,7 +1599,7 @@ impl<C> Scheduler<C> {
         info!("Scheduler::gracefully_stop(): slot: {} id_{:016x} durations 2/2 (wall): scheduler: {}us, error_collector: {}us, lanes: {}us = {:?}", self.slot.map(|s| format!("{}", s)).unwrap_or("-".into()), self.random_id, scheduler_thread_wall_time_us, error_collector_thread_wall_time_us, executing_thread_wall_time_us.iter().sum::<u128>(), &executing_thread_wall_time_us);
         */
 
-        Ok(Some(r))
+        Ok(())
     }
 
     fn handle_aborted_executions(&self) -> Vec<Result<Option<C>>> {

From cc3485ce3c2562c1ee882c702b86af1c4a1fdf39 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 22 Sep 2022 09:55:24 +0900
Subject: [PATCH 3099/3199] save

---
 runtime/src/bank.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 9e3b1bdeee356a..d251aa02c41a4b 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1574,7 +1574,7 @@ impl<C> Scheduler<C> {
             .unwrap();
         checkpoint.wait_for_restart(None);
         let r = checkpoint.take_restart_value();
-        self.collected_results.push(r);
+        self.collected_results.lock().unwrap().push(r);
         {
             *self.bank.write().unwrap() = None;
             self.slot.store(0, std::sync::atomic::Ordering::SeqCst);

From 9b679ad53938a55d3881728ea115019815a29e67 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 22 Sep 2022 09:55:56 +0900
Subject: [PATCH 3100/3199] save

---
 runtime/src/bank.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index d251aa02c41a4b..5e2a1795759600 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1574,7 +1574,7 @@ impl<C> Scheduler<C> {
             .unwrap();
         checkpoint.wait_for_restart(None);
         let r = checkpoint.take_restart_value();
-        self.collected_results.lock().unwrap().push(r);
+        self.collected_results.lock().unwrap().push(Ok(Some(r)));
         {
             *self.bank.write().unwrap() = None;
             self.slot.store(0, std::sync::atomic::Ordering::SeqCst);

From 1c5b187ef1af35d10afb35362d98c0c380489108 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 22 Sep 2022 09:57:51 +0900
Subject: [PATCH 3101/3199] save

---
 runtime/src/bank.rs | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 5e2a1795759600..ba112846b1ec13 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -6724,11 +6724,13 @@ impl Bank {
         }
     }
 
+    /*
     pub fn handle_aborted_transactions(&self) -> Vec<Result<Option<ExecuteTimings>>> {
         let s = self.scheduler2.read().unwrap();
         let scheduler = s.as_ref().unwrap();
         scheduler.handle_aborted_executions()
     }
+    */ 
 
     /// Process a batch of transactions.
     #[must_use]

From 9d1cc8fd1a8c18f264c5b847320e44dc359a84b3 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 22 Sep 2022 10:00:33 +0900
Subject: [PATCH 3102/3199] save

---
 runtime/src/bank.rs | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index ba112846b1ec13..99e411c1ffd3d4 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1447,15 +1447,15 @@ impl Scheduler<ExecuteTimings> {
                                 info!("execute_substage: slot: {} transaction_index: {} timings: {:?}", ee.slot, ee.transaction_index, timings);
                             }
 
-                            if ee.is_aborted() {
+                            if let Some(Err(e)) = ee.execution_result.take() {
                                 warn!(
                                     "scheduler: Unexpected validator error: {:?}, transaction: {:?}",
-                                    ee.execution_result, ee.task.tx.0
+                                    e, ee.task.tx.0
                                 );
                                 collected_results_in_collector_thread
                                     .lock()
                                     .unwrap()
-                                    .push(ee.execution_result.take().unwrap().map(|a| Default::default()));
+                                    .push(Err(e));
                             }
                             drop(ee);
                         },

From 34d86c6c526ee715d4d344e70d501aa12a62260f Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 22 Sep 2022 10:01:22 +0900
Subject: [PATCH 3103/3199] save

---
 runtime/src/bank.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 99e411c1ffd3d4..96527f2b9aee07 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1258,7 +1258,7 @@ struct Scheduler<C> {
     transaction_sender: Option<crossbeam_channel::Sender<solana_scheduler::SchedulablePayload<C>>>,
     preloader: Arc<solana_scheduler::Preloader>,
     graceful_stop_initiated: AtomicBool,
-    collected_results: Arc<std::sync::Mutex<Vec<Result<Option<C>>>>>,
+    collected_results: Arc<std::sync::Mutex<Vec<Result<C>>>>,
     bank: std::sync::Arc<std::sync::RwLock<std::option::Option<std::sync::Weak<Bank>>>>,
     slot: AtomicU64,
 }

From 3372b5a670fd5d76c38a51a36f4ce98cb93409ab Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 22 Sep 2022 10:01:34 +0900
Subject: [PATCH 3104/3199] save

---
 runtime/src/bank.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 96527f2b9aee07..c2a68a1bc797f6 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1574,7 +1574,7 @@ impl<C> Scheduler<C> {
             .unwrap();
         checkpoint.wait_for_restart(None);
         let r = checkpoint.take_restart_value();
-        self.collected_results.lock().unwrap().push(Ok(Some(r)));
+        self.collected_results.lock().unwrap().push(Ok(r));
         {
             *self.bank.write().unwrap() = None;
             self.slot.store(0, std::sync::atomic::Ordering::SeqCst);

From 0e9d53888e067dd1772672d0acc05cfae5b44db5 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 22 Sep 2022 10:02:38 +0900
Subject: [PATCH 3105/3199] save

---
 runtime/src/bank.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index c2a68a1bc797f6..9e971c34dd4e30 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -8382,7 +8382,7 @@ impl Bank {
         total_accounts_stats
     }
 
-    pub fn wait_for_scheduler(&self, via_drop: bool) -> Result<Option<ExecuteTimings>> {
+    pub fn wait_for_scheduler(&self, via_drop: bool) -> Result<ExecuteTimings> {
         let s: Option<Arc<Scheduler<ExecuteTimings>>> = self.scheduler2.write().unwrap().take();
 
         if let Some(scheduler) = s {
@@ -8400,7 +8400,7 @@ impl Bank {
                 via_drop, current_thread_name
             );
 
-            Ok(None)
+            Ok(Default::default())
         }
     }
 }

From 21cc910f6f902f7d540cadd68a82a97b1ed25bf0 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 22 Sep 2022 10:03:17 +0900
Subject: [PATCH 3106/3199] save

---
 runtime/src/bank.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 9e971c34dd4e30..9aae92785532fd 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1602,7 +1602,7 @@ impl<C> Scheduler<C> {
         Ok(())
     }
 
-    fn handle_aborted_executions(&self) -> Vec<Result<Option<C>>> {
+    fn handle_aborted_executions(&self) -> Vec<Result<C>> {
         std::mem::take(&mut self.collected_results.lock().unwrap())
     }
 }

From 356974297f75c50d949ea978e951da3f8c745a95 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 22 Sep 2022 12:59:07 +0900
Subject: [PATCH 3107/3199] save

---
 core/src/replay_stage.rs | 1 +
 1 file changed, 1 insertion(+)

diff --git a/core/src/replay_stage.rs b/core/src/replay_stage.rs
index 6401f1a919a0a5..83a7796fd07ec2 100644
--- a/core/src/replay_stage.rs
+++ b/core/src/replay_stage.rs
@@ -2506,6 +2506,7 @@ impl ReplayStage {
 
                 let replay_stats = bank_progress.replay_stats.clone();
                 let r_replay_stats = replay_stats.read().unwrap();
+                r_replay_stats.process_execute_batches_internal_metrics(3);
                 let replay_progress = bank_progress.replay_progress.clone();
                 let r_replay_progress = replay_progress.read().unwrap();
                 debug!("bank {} is completed replay from blockstore, contribute to update cost with {:?}",

From 63001dfa4863caa9f204c1350b6f75c5b769e7ad Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 22 Sep 2022 13:01:04 +0900
Subject: [PATCH 3108/3199] save

---
 core/src/replay_stage.rs | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/core/src/replay_stage.rs b/core/src/replay_stage.rs
index 83a7796fd07ec2..9538f5c7c5f3f3 100644
--- a/core/src/replay_stage.rs
+++ b/core/src/replay_stage.rs
@@ -2506,7 +2506,9 @@ impl ReplayStage {
 
                 let replay_stats = bank_progress.replay_stats.clone();
                 let r_replay_stats = replay_stats.read().unwrap();
-                r_replay_stats.process_execute_batches_internal_metrics(3);
+                let mut metrics = ExecuteBatchesInternalMetrics::default();
+                metrics.execution_timings_per_thread.insert(0, r);
+                r_replay_stats.process_execute_batches_internal_metrics(metrics);
                 let replay_progress = bank_progress.replay_progress.clone();
                 let r_replay_progress = replay_progress.read().unwrap();
                 debug!("bank {} is completed replay from blockstore, contribute to update cost with {:?}",

From a6baac3b25b7acfe7e7617f4c15f36c50fcd59e6 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 22 Sep 2022 13:01:32 +0900
Subject: [PATCH 3109/3199] save

---
 ledger/src/blockstore_processor.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/ledger/src/blockstore_processor.rs b/ledger/src/blockstore_processor.rs
index 143efdd8e08a47..f370e4fa2bfaa8 100644
--- a/ledger/src/blockstore_processor.rs
+++ b/ledger/src/blockstore_processor.rs
@@ -266,7 +266,7 @@ fn execute_batch(
 }
 
 #[derive(Default)]
-struct ExecuteBatchesInternalMetrics {
+pub struct ExecuteBatchesInternalMetrics {
     execution_timings_per_thread: HashMap<usize, ThreadExecuteTimings>,
     total_batches_len: u64,
     execute_batches_us: u64,
@@ -796,7 +796,7 @@ pub struct ConfirmationTiming {
 }
 
 impl ConfirmationTiming {
-    fn process_execute_batches_internal_metrics(
+    pub fn process_execute_batches_internal_metrics(
         &mut self,
         execute_batches_internal_metrics: ExecuteBatchesInternalMetrics,
     ) {

From 03bf0d3c60aa7d4bd3b411818b4284c17268eed8 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 22 Sep 2022 13:01:58 +0900
Subject: [PATCH 3110/3199] save

---
 core/src/replay_stage.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/core/src/replay_stage.rs b/core/src/replay_stage.rs
index 9538f5c7c5f3f3..aa98fdfb0d19df 100644
--- a/core/src/replay_stage.rs
+++ b/core/src/replay_stage.rs
@@ -2506,7 +2506,7 @@ impl ReplayStage {
 
                 let replay_stats = bank_progress.replay_stats.clone();
                 let r_replay_stats = replay_stats.read().unwrap();
-                let mut metrics = ExecuteBatchesInternalMetrics::default();
+                let mut metrics = solana_ledger::blockstore_processor::ExecuteBatchesInternalMetrics::default();
                 metrics.execution_timings_per_thread.insert(0, r);
                 r_replay_stats.process_execute_batches_internal_metrics(metrics);
                 let replay_progress = bank_progress.replay_progress.clone();

From d68106f6147c3dfb87cbed350ae1a513e799e10f Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 22 Sep 2022 13:02:38 +0900
Subject: [PATCH 3111/3199] save

---
 core/src/replay_stage.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/core/src/replay_stage.rs b/core/src/replay_stage.rs
index aa98fdfb0d19df..31247273337567 100644
--- a/core/src/replay_stage.rs
+++ b/core/src/replay_stage.rs
@@ -2477,7 +2477,7 @@ impl ReplayStage {
             assert_eq!(bank_slot, bank.slot());
             if bank.is_complete() {
                 info!("waiting for completed bank: slot: {}", bank.slot());
-                let r: usize = bank.wait_for_scheduler(false).unwrap();
+                let cumulative_timings = bank.wait_for_scheduler(false).unwrap();
                 if let Err(err) = r {
                     // Error means the slot needs to be marked as dead
                     Self::mark_dead_slot(
@@ -2507,7 +2507,7 @@ impl ReplayStage {
                 let replay_stats = bank_progress.replay_stats.clone();
                 let r_replay_stats = replay_stats.read().unwrap();
                 let mut metrics = solana_ledger::blockstore_processor::ExecuteBatchesInternalMetrics::default();
-                metrics.execution_timings_per_thread.insert(0, r);
+                metrics.execution_timings_per_thread.insert(0, cumulative_timings);
                 r_replay_stats.process_execute_batches_internal_metrics(metrics);
                 let replay_progress = bank_progress.replay_progress.clone();
                 let r_replay_progress = replay_progress.read().unwrap();

From dc10b4f3ddd250ddd054a66c1e5c981e2469f1e3 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 22 Sep 2022 13:02:56 +0900
Subject: [PATCH 3112/3199] save

---
 core/src/replay_stage.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/core/src/replay_stage.rs b/core/src/replay_stage.rs
index 31247273337567..9366141ed34b17 100644
--- a/core/src/replay_stage.rs
+++ b/core/src/replay_stage.rs
@@ -2478,7 +2478,7 @@ impl ReplayStage {
             if bank.is_complete() {
                 info!("waiting for completed bank: slot: {}", bank.slot());
                 let cumulative_timings = bank.wait_for_scheduler(false).unwrap();
-                if let Err(err) = r {
+                if let Err(err) = cumulative_timings {
                     // Error means the slot needs to be marked as dead
                     Self::mark_dead_slot(
                         blockstore,

From 4d7a8e3cb6c7db5f3bb2b4ed02ac3694c84e7348 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 22 Sep 2022 13:03:14 +0900
Subject: [PATCH 3113/3199] save

---
 core/src/replay_stage.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/core/src/replay_stage.rs b/core/src/replay_stage.rs
index 9366141ed34b17..16b9ba34cc7895 100644
--- a/core/src/replay_stage.rs
+++ b/core/src/replay_stage.rs
@@ -2477,7 +2477,7 @@ impl ReplayStage {
             assert_eq!(bank_slot, bank.slot());
             if bank.is_complete() {
                 info!("waiting for completed bank: slot: {}", bank.slot());
-                let cumulative_timings = bank.wait_for_scheduler(false).unwrap();
+                let cumulative_timings = bank.wait_for_scheduler(false);
                 if let Err(err) = cumulative_timings {
                     // Error means the slot needs to be marked as dead
                     Self::mark_dead_slot(

From 489671f7db089cc233e897ba0ca5def203f181c1 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 22 Sep 2022 13:04:13 +0900
Subject: [PATCH 3114/3199] save

---
 ledger/src/blockstore_processor.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ledger/src/blockstore_processor.rs b/ledger/src/blockstore_processor.rs
index f370e4fa2bfaa8..8bd94c59e81a39 100644
--- a/ledger/src/blockstore_processor.rs
+++ b/ledger/src/blockstore_processor.rs
@@ -267,7 +267,7 @@ fn execute_batch(
 
 #[derive(Default)]
 pub struct ExecuteBatchesInternalMetrics {
-    execution_timings_per_thread: HashMap<usize, ThreadExecuteTimings>,
+    pub execution_timings_per_thread: HashMap<usize, ThreadExecuteTimings>,
     total_batches_len: u64,
     execute_batches_us: u64,
 }

From 8a2e798cb3eb0b61c84e18e5229e5b847eabb67d Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 22 Sep 2022 13:05:09 +0900
Subject: [PATCH 3115/3199] save

---
 core/src/replay_stage.rs | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/core/src/replay_stage.rs b/core/src/replay_stage.rs
index 16b9ba34cc7895..eb4b42e455cb4d 100644
--- a/core/src/replay_stage.rs
+++ b/core/src/replay_stage.rs
@@ -2498,6 +2498,7 @@ impl ReplayStage {
                     // bank is completed
                     continue;
                 }
+                let cumulative_timings2 = cumulative_timings.unwrap();
 
                 let mut bank_complete_time = Measure::start("bank_complete_time");
                 let bank_progress = progress
@@ -2507,7 +2508,7 @@ impl ReplayStage {
                 let replay_stats = bank_progress.replay_stats.clone();
                 let r_replay_stats = replay_stats.read().unwrap();
                 let mut metrics = solana_ledger::blockstore_processor::ExecuteBatchesInternalMetrics::default();
-                metrics.execution_timings_per_thread.insert(0, cumulative_timings);
+                metrics.execution_timings_per_thread.insert(0, cumulative_timings2);
                 r_replay_stats.process_execute_batches_internal_metrics(metrics);
                 let replay_progress = bank_progress.replay_progress.clone();
                 let r_replay_progress = replay_progress.read().unwrap();

From 30cded868aad65449d399f98ba67c87935824be2 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 22 Sep 2022 13:06:18 +0900
Subject: [PATCH 3116/3199] save

---
 core/src/replay_stage.rs | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/core/src/replay_stage.rs b/core/src/replay_stage.rs
index eb4b42e455cb4d..0fb55f9d124f40 100644
--- a/core/src/replay_stage.rs
+++ b/core/src/replay_stage.rs
@@ -2498,7 +2498,10 @@ impl ReplayStage {
                     // bank is completed
                     continue;
                 }
-                let cumulative_timings2 = cumulative_timings.unwrap();
+                let cumulative_timings2 = ThreadExecuteTimings {
+                    execute_timings: cumulative_timings.unwrap(),
+                    ..Default::default()
+                };
 
                 let mut bank_complete_time = Measure::start("bank_complete_time");
                 let bank_progress = progress

From 825aa06bc6c7b36373a5c333fa461862d5ed98c5 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 22 Sep 2022 13:06:35 +0900
Subject: [PATCH 3117/3199] save

---
 core/src/replay_stage.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/core/src/replay_stage.rs b/core/src/replay_stage.rs
index 0fb55f9d124f40..f378c8f5876242 100644
--- a/core/src/replay_stage.rs
+++ b/core/src/replay_stage.rs
@@ -2498,7 +2498,7 @@ impl ReplayStage {
                     // bank is completed
                     continue;
                 }
-                let cumulative_timings2 = ThreadExecuteTimings {
+                let cumulative_timings2 = solana_program_runtime::timings::ThreadExecuteTimings {
                     execute_timings: cumulative_timings.unwrap(),
                     ..Default::default()
                 };

From bbda77cae09b2b2af372e2d1603c1196c24925fa Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 22 Sep 2022 13:08:36 +0900
Subject: [PATCH 3118/3199] save

---
 core/src/replay_stage.rs | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/core/src/replay_stage.rs b/core/src/replay_stage.rs
index f378c8f5876242..c88ee457bb8b66 100644
--- a/core/src/replay_stage.rs
+++ b/core/src/replay_stage.rs
@@ -2513,8 +2513,9 @@ impl ReplayStage {
                 let mut metrics = solana_ledger::blockstore_processor::ExecuteBatchesInternalMetrics::default();
                 metrics.execution_timings_per_thread.insert(0, cumulative_timings2);
                 r_replay_stats.process_execute_batches_internal_metrics(metrics);
+
                 let replay_progress = bank_progress.replay_progress.clone();
-                let r_replay_progress = replay_progress.read().unwrap();
+                let w_replay_progress = replay_progress.write().unwrap();
                 debug!("bank {} is completed replay from blockstore, contribute to update cost with {:?}",
                     bank.slot(),
                     r_replay_stats.execute_timings
@@ -2599,8 +2600,8 @@ impl ReplayStage {
 
                 r_replay_stats.report_stats(
                     bank.slot(),
-                    r_replay_progress.num_entries,
-                    r_replay_progress.num_shreds,
+                    w_replay_progress.num_entries,
+                    w_replay_progress.num_shreds,
                     bank_complete_time.as_us(),
                 );
                 execute_timings.accumulate(&r_replay_stats.execute_timings);

From 37566f67f041dc16bd86d1e503d317f4f9fdc486 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 22 Sep 2022 13:09:20 +0900
Subject: [PATCH 3119/3199] save

---
 core/src/replay_stage.rs | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/core/src/replay_stage.rs b/core/src/replay_stage.rs
index c88ee457bb8b66..c914439824a8a7 100644
--- a/core/src/replay_stage.rs
+++ b/core/src/replay_stage.rs
@@ -2509,13 +2509,13 @@ impl ReplayStage {
                     .expect("Bank fork progress entry missing for completed bank");
 
                 let replay_stats = bank_progress.replay_stats.clone();
-                let r_replay_stats = replay_stats.read().unwrap();
+                let r_replay_stats = replay_stats.write().unwrap();
                 let mut metrics = solana_ledger::blockstore_processor::ExecuteBatchesInternalMetrics::default();
                 metrics.execution_timings_per_thread.insert(0, cumulative_timings2);
                 r_replay_stats.process_execute_batches_internal_metrics(metrics);
 
                 let replay_progress = bank_progress.replay_progress.clone();
-                let w_replay_progress = replay_progress.write().unwrap();
+                let r_replay_progress = replay_progress.read().unwrap();
                 debug!("bank {} is completed replay from blockstore, contribute to update cost with {:?}",
                     bank.slot(),
                     r_replay_stats.execute_timings
@@ -2600,8 +2600,8 @@ impl ReplayStage {
 
                 r_replay_stats.report_stats(
                     bank.slot(),
-                    w_replay_progress.num_entries,
-                    w_replay_progress.num_shreds,
+                    r_replay_progress.num_entries,
+                    r_replay_progress.num_shreds,
                     bank_complete_time.as_us(),
                 );
                 execute_timings.accumulate(&r_replay_stats.execute_timings);

From 77b95bbf4085e523444c3d891a43d53a7db1f2b5 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 22 Sep 2022 13:09:35 +0900
Subject: [PATCH 3120/3199] save

---
 core/src/replay_stage.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/core/src/replay_stage.rs b/core/src/replay_stage.rs
index c914439824a8a7..990ad775e7ab4c 100644
--- a/core/src/replay_stage.rs
+++ b/core/src/replay_stage.rs
@@ -2509,7 +2509,7 @@ impl ReplayStage {
                     .expect("Bank fork progress entry missing for completed bank");
 
                 let replay_stats = bank_progress.replay_stats.clone();
-                let r_replay_stats = replay_stats.write().unwrap();
+                let mut r_replay_stats = replay_stats.write().unwrap();
                 let mut metrics = solana_ledger::blockstore_processor::ExecuteBatchesInternalMetrics::default();
                 metrics.execution_timings_per_thread.insert(0, cumulative_timings2);
                 r_replay_stats.process_execute_batches_internal_metrics(metrics);

From 9a5e05d4440249ae17c68e24d9814b140c39316d Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 22 Sep 2022 13:10:40 +0900
Subject: [PATCH 3121/3199] save

---
 ledger-tool/src/main.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ledger-tool/src/main.rs b/ledger-tool/src/main.rs
index 1b62e26d1b488e..8d49af5e4e6611 100644
--- a/ledger-tool/src/main.rs
+++ b/ledger-tool/src/main.rs
@@ -259,7 +259,7 @@ fn output_slot(
         }
     }
 
-    let (muxed_sender, muxed_receiver) = crossbeam_channel::unbounded();
+    let (muxed_sender, muxed_receiver) = crossbeam_channel::unbounded::<solana_scheduler::SchedulablePayload<()>>();
 
     // this should be target number of saturated cpu cores
     let lane_count = std::env::var("EXECUTION_LANE_COUNT")

From 99398f02371c036ce39087426f1ee76af29ca3c1 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 22 Sep 2022 15:04:31 +0900
Subject: [PATCH 3122/3199] save

---
 ledger/src/blockstore_processor.rs |  4 ++--
 runtime/src/bank.rs                |  4 ++--
 scheduler/src/lib.rs               | 22 +++++++++++-----------
 3 files changed, 15 insertions(+), 15 deletions(-)

diff --git a/ledger/src/blockstore_processor.rs b/ledger/src/blockstore_processor.rs
index 8bd94c59e81a39..9e3bebc3fea387 100644
--- a/ledger/src/blockstore_processor.rs
+++ b/ledger/src/blockstore_processor.rs
@@ -420,7 +420,7 @@ fn process_entries_with_callback(
                 // If it's a tick, save it for later
                 tick_hashes.push(hash);
                 if bank.is_block_boundary(bank.tick_height() + tick_hashes.len() as u64) {
-                    info!("process_entries_with_callback(): slot: {} idx: {} scheduled {} txs so far in {}us (still in loop)", bank.slot(), starting_index, tx_count, started.elapsed().as_micros());
+                    trace!("process_entries_with_callback(): slot: {} idx: {} scheduled {} txs so far in {}us (still in loop)", bank.slot(), starting_index, tx_count, started.elapsed().as_micros());
                     started = std::time::Instant::now();
                     tx_count = 0;
 
@@ -450,7 +450,7 @@ fn process_entries_with_callback(
             }
         }
     }
-    info!(
+    trace!(
         "process_entries_with_callback(): slot: {} scheduled {} txs so far in {}us (after loop)",
         bank.slot(),
         tx_count,
diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 9aae92785532fd..0df1df4c42146b 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -4290,7 +4290,7 @@ impl Bank {
     /// reaches its max tick height. Can be called by tests to get new blockhashes for transaction
     /// processing without advancing to a new bank slot.
     pub fn register_recent_blockhash(&self, blockhash: &Hash) {
-        info!(
+        debug!(
             "register_recent_blockhash: slot: {} reinitializing the scheduler: start",
             self.slot()
         );
@@ -4320,7 +4320,7 @@ impl Bank {
                 .push(last_result);
         //*self.scheduler.write().unwrap() = new_scheduler;
 
-        info!(
+        debug!(
             "register_recent_blockhash: slot: {} reinitializing the scheduler: end",
             self.slot()
         );
diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 8d54009643e3ef..a115e6db800421 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -2019,31 +2019,31 @@ impl<T> Checkpoint<T> {
     pub fn wait_for_restart(&self, maybe_given_restart_value: Option<T>) {
         let current_thread_name = std::thread::current().name().unwrap().to_string();
         let mut g = self.0.lock().unwrap();
-        let (self_remaining_threads_guard, self_return_value) = &mut *g;
-        info!(
+        let (self_remaining_threads, self_return_value) = &mut *g;
+        trace!(
             "Checkpoint::wait_for_restart: {} is entering at {} -> {}",
             current_thread_name,
-            *self_remaining_threads_guard,
-            *self_remaining_threads_guard - 1
+            *self_remaining_threads,
+            *self_remaining_threads - 1
         );
 
-        *self_remaining_threads_guard -= 1;
+        *self_remaining_threads -= 1;
 
         if let Some(given_restart_value) = maybe_given_restart_value {
             assert!(self_return_value.is_none());
             *self_return_value = Some(given_restart_value);
         }
 
-        if *self_remaining_threads_guard == 0 {
+        if *self_remaining_threads == 0 {
             assert!(self_return_value.is_some());
-            drop(self_remaining_threads_guard);
+            drop(self_remaining_threads);
             self.1.notify_all();
-            info!(
+            trace!(
                 "Checkpoint::wait_for_restart: {} notified all others...",
                 current_thread_name
             );
         } else {
-            info!(
+            trace!(
                 "Checkpoint::wait_for_restart: {} is paused...",
                 current_thread_name
             );
@@ -2053,7 +2053,7 @@ impl<T> Checkpoint<T> {
                     remaining_threads > 0
                 })
                 .unwrap();
-            info!(
+            trace!(
                 "Checkpoint::wait_for_restart: {} is started...",
                 current_thread_name
             );
@@ -2062,7 +2062,7 @@ impl<T> Checkpoint<T> {
 
     pub fn take_restart_value(&self) -> T {
         let mut g = self.0.lock().unwrap();
-        let (self_remaining_threads_guard, self_return_value) = &mut *g;
+        let (self_remaining_threads, self_return_value) = &mut *g;
         self_return_value.take().unwrap()
     }
 

From 5407c6bb7625961d104d10eb1960a155423e0a69 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 22 Sep 2022 15:05:08 +0900
Subject: [PATCH 3123/3199] save

---
 runtime/src/bank.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 0df1df4c42146b..d0cf0a0b8e57f4 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1212,7 +1212,7 @@ impl SchedulerPool<ExecuteTimings> {
 
     fn take_from_pool(&mut self) -> Arc<Scheduler<ExecuteTimings>> {
         if let Some(scheduler) = self.schedulers.pop() {
-            info!(
+            trace!(
                 "SchedulerPool: id_{:016x} is taken... len: {} => {}",
                 scheduler.random_id,
                 self.schedulers.len() + 1,
@@ -1226,7 +1226,7 @@ impl SchedulerPool<ExecuteTimings> {
     }
 
     fn return_to_pool(&mut self, scheduler: Arc<Scheduler<ExecuteTimings>>) {
-        info!(
+        trace!(
             "SchedulerPool: id_{:016x} is returned... len: {} => {}",
             scheduler.random_id,
             self.schedulers.len(),

From 829bbade4b12817e150c18cf0e418eec85fc77fc Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 22 Sep 2022 15:07:09 +0900
Subject: [PATCH 3124/3199] save

---
 runtime/src/bank.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index d0cf0a0b8e57f4..152d789831bbb7 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -6704,7 +6704,7 @@ impl Bank {
                 drop(ss);
 
                 let s = self.scheduler2.read().unwrap();
-                info!("reconfigured scheduler to the bank slot: {}", self.slot());
+                trace!("reconfigured scheduler to the bank slot: {}", self.slot());
                 s
             } else {
                 assert_eq!(

From 1d0d92c14835e0cafb41a9b312dfc3bdb292c60a Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 22 Sep 2022 15:18:56 +0900
Subject: [PATCH 3125/3199] save

---
 core/src/replay_stage.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/core/src/replay_stage.rs b/core/src/replay_stage.rs
index 990ad775e7ab4c..cd89cccf77222a 100644
--- a/core/src/replay_stage.rs
+++ b/core/src/replay_stage.rs
@@ -916,7 +916,7 @@ impl ReplayStage {
                     if !did_complete_bank {
                         // only wait for the signal if we did not just process a bank; maybe there are more slots available
 
-                        let timer = Duration::from_millis(2000);
+                        let timer = Duration::from_millis(100);
                         let result = ledger_signal_receiver.recv_timeout(timer);
                         match result {
                             Err(RecvTimeoutError::Timeout) => (),

From bb198919815701bb82dcfb7e45f97e98b1401b6f Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 22 Sep 2022 15:56:37 +0900
Subject: [PATCH 3126/3199] save

---
 runtime/src/bank.rs  | 2 +-
 scheduler/src/lib.rs | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 152d789831bbb7..2870ddcc51d180 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1557,7 +1557,7 @@ impl<C> Scheduler<C> {
         self.graceful_stop_initiated
             .store(true, std::sync::atomic::Ordering::SeqCst);
 
-        info!(
+        trace!(
             "Scheduler::gracefully_stop(): id_{:016x} waiting..",
             self.random_id
         );
diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index a115e6db800421..2a6badc91f87a4 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -2062,7 +2062,7 @@ impl<T> Checkpoint<T> {
 
     pub fn take_restart_value(&self) -> T {
         let mut g = self.0.lock().unwrap();
-        let (self_remaining_threads, self_return_value) = &mut *g;
+        let (_self_remaining_threads, self_return_value) = &mut *g;
         self_return_value.take().unwrap()
     }
 

From c365e7437e1ab386f2fe670ace220f866d78bb31 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 22 Sep 2022 22:00:23 +0900
Subject: [PATCH 3127/3199] save

---
 core/src/replay_stage.rs |  7 +++-
 ledger-tool/src/main.rs  |  3 +-
 runtime/src/bank.rs      | 22 +++++-----
 scheduler/src/lib.rs     | 91 ++++++++++++++++++++++------------------
 4 files changed, 69 insertions(+), 54 deletions(-)

diff --git a/core/src/replay_stage.rs b/core/src/replay_stage.rs
index cd89cccf77222a..0434088b6b7ace 100644
--- a/core/src/replay_stage.rs
+++ b/core/src/replay_stage.rs
@@ -2510,8 +2510,11 @@ impl ReplayStage {
 
                 let replay_stats = bank_progress.replay_stats.clone();
                 let mut r_replay_stats = replay_stats.write().unwrap();
-                let mut metrics = solana_ledger::blockstore_processor::ExecuteBatchesInternalMetrics::default();
-                metrics.execution_timings_per_thread.insert(0, cumulative_timings2);
+                let mut metrics =
+                    solana_ledger::blockstore_processor::ExecuteBatchesInternalMetrics::default();
+                metrics
+                    .execution_timings_per_thread
+                    .insert(0, cumulative_timings2);
                 r_replay_stats.process_execute_batches_internal_metrics(metrics);
 
                 let replay_progress = bank_progress.replay_progress.clone();
diff --git a/ledger-tool/src/main.rs b/ledger-tool/src/main.rs
index 8d49af5e4e6611..57bda5bf6d2088 100644
--- a/ledger-tool/src/main.rs
+++ b/ledger-tool/src/main.rs
@@ -259,7 +259,8 @@ fn output_slot(
         }
     }
 
-    let (muxed_sender, muxed_receiver) = crossbeam_channel::unbounded::<solana_scheduler::SchedulablePayload<()>>();
+    let (muxed_sender, muxed_receiver) =
+        crossbeam_channel::unbounded::<solana_scheduler::SchedulablePayload<()>>();
 
     // this should be target number of saturated cpu cores
     let lane_count = std::env::var("EXECUTION_LANE_COUNT")
diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 2870ddcc51d180..5133e178eb9e52 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1207,7 +1207,8 @@ impl SchedulerPool<ExecuteTimings> {
     }
 
     fn create(&mut self) {
-        self.schedulers.push(Arc::new(Scheduler::<ExecuteTimings>::default2()));
+        self.schedulers
+            .push(Arc::new(Scheduler::<ExecuteTimings>::default2()));
     }
 
     fn take_from_pool(&mut self) -> Arc<Scheduler<ExecuteTimings>> {
@@ -4311,13 +4312,13 @@ impl Bank {
                 last_result
             );
         }
-            //new_scheduler.collected_results.lock().unwrap().push(maybe_last_error);
-            s2.as_ref()
-                .unwrap()
-                .collected_results
-                .lock()
-                .unwrap()
-                .push(last_result);
+        //new_scheduler.collected_results.lock().unwrap().push(maybe_last_error);
+        s2.as_ref()
+            .unwrap()
+            .collected_results
+            .lock()
+            .unwrap()
+            .push(last_result);
         //*self.scheduler.write().unwrap() = new_scheduler;
 
         debug!(
@@ -6730,7 +6731,7 @@ impl Bank {
         let scheduler = s.as_ref().unwrap();
         scheduler.handle_aborted_executions()
     }
-    */ 
+    */
 
     /// Process a batch of transactions.
     #[must_use]
@@ -8390,7 +8391,8 @@ impl Bank {
             let e = scheduler
                 .handle_aborted_executions()
                 .into_iter()
-                .next().unwrap();
+                .next()
+                .unwrap();
             SCHEDULER_POOL.lock().unwrap().return_to_pool(scheduler);
             e
         } else {
diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 2a6badc91f87a4..2b5933c096f9a8 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -74,7 +74,10 @@ impl ExecutionEnvironment {
             .load(std::sync::atomic::Ordering::SeqCst)
             > 0;
         for lock_attempt in self.finalized_lock_attempts.iter_mut() {
-            let ll = lock_attempt.target_page_mut(ast).task_ids.reindex(should_remove, &uq);
+            let ll = lock_attempt
+                .target_page_mut(ast)
+                .task_ids
+                .reindex(should_remove, &uq);
             if let Some(heaviest_uncontended) = ll {
                 lock_attempt.heaviest_uncontended = Some(heaviest_uncontended);
             };
@@ -154,7 +157,7 @@ impl LockAttempt {
     */
 
     pub fn target_contended_write_task_count(&self) -> &std::sync::atomic::AtomicUsize {
-        &self.target.0.1
+        &self.target.0 .1
     }
 
     fn target_page_mut<AST: AtScheduleThread>(&self, ast: AST) -> std::cell::RefMut<'_, Page> {
@@ -231,16 +234,12 @@ impl SkipListTaskIds {
     }
 
     pub fn heaviest_task_id(&self) -> Option<TaskId> {
-        self
-            .task_ids
-            .back()
-            .map(|j| *j.key())
+        self.task_ids.back().map(|j| *j.key())
     }
 
     #[inline(never)]
     fn reindex(&self, should_remove: bool, uq: &UniqueWeight) -> Option<TaskInQueue> {
-        self
-            .heaviest_task_cursor()
+        self.heaviest_task_cursor()
             .map(|mut task_cursor| {
                 let mut found = true;
                 let mut removed = false;
@@ -287,17 +286,12 @@ impl BTreeMapTaskIds {
     }
 
     #[inline(never)]
-    fn heaviest_task_cursor(
-        &self,
-    ) -> impl Iterator<Item = &TaskInQueue> {
+    fn heaviest_task_cursor(&self) -> impl Iterator<Item = &TaskInQueue> {
         self.task_ids.values().rev()
     }
 
     pub fn heaviest_task_id(&mut self) -> Option<TaskId> {
-        self
-            .task_ids
-            .last_entry()
-            .map(|j| *j.key())
+        self.task_ids.last_entry().map(|j| *j.key())
     }
 
     #[inline(never)]
@@ -306,8 +300,7 @@ impl BTreeMapTaskIds {
             self.remove_task(uq);
         }
 
-        self
-            .heaviest_task_cursor()
+        self.heaviest_task_cursor()
             .find(|task| {
                 assert!(!task.already_finished());
                 task.currently_contended()
@@ -409,7 +402,8 @@ impl AddressBook {
     ) -> CU {
         let tcuw = attempt
             //.target_contended_unique_weights()
-            .target_page_mut(ast).task_ids
+            .target_page_mut(ast)
+            .task_ids
             .heaviest_task_id();
 
         let strictly_lockable_for_replay = if tcuw.is_none() {
@@ -834,7 +828,8 @@ impl Task {
     ) {
         for lock_attempt in &*this.lock_attempts_mut(ast) {
             lock_attempt
-                .target_page_mut(ast).task_ids
+                .target_page_mut(ast)
+                .task_ids
                 .insert_task(this.unique_weight, Task::clone_in_queue(this));
 
             if lock_attempt.requested_usage == RequestedUsage::Writable {
@@ -914,7 +909,11 @@ struct ChannelBackedTaskQueue<'a, C> {
 
 impl<'a, C> ChannelBackedTaskQueue<'a, C> {
     fn new(channel: &'a crossbeam_channel::Receiver<SchedulablePayload<C>>) -> Self {
-        Self {channel, buffered_task: None, buffered_flush: None}
+        Self {
+            channel,
+            buffered_task: None,
+            buffered_flush: None,
+        }
     }
 
     fn buffer(&mut self, task: TaskInQueue) {
@@ -923,10 +922,11 @@ impl<'a, C> ChannelBackedTaskQueue<'a, C> {
     }
 
     fn task_count_hint(&self) -> usize {
-        self.channel.len() + (match self.buffered_task {
-            None => 0,
-            Some(_) => 1,
-        })
+        self.channel.len()
+            + (match self.buffered_task {
+                None => 0,
+                Some(_) => 1,
+            })
     }
 
     fn has_no_task_hint(&self) -> bool {
@@ -945,8 +945,14 @@ impl<'a, C> ChannelBackedTaskQueue<'a, C> {
                 // unblocking recv must have been gurantted to succeed at the time of this method
                 // invocation
                 match self.channel.try_recv().unwrap() {
-                    SchedulablePayload(Flushable::Payload(task)) => Some(ChannelBackedTaskQueueEntry(task)),
-                    SchedulablePayload(Flushable::Flush(f)) => { assert!(self.buffered_flush.is_none()); self.buffered_flush = Some(f); None }
+                    SchedulablePayload(Flushable::Payload(task)) => {
+                        Some(ChannelBackedTaskQueueEntry(task))
+                    }
+                    SchedulablePayload(Flushable::Flush(f)) => {
+                        assert!(self.buffered_flush.is_none());
+                        self.buffered_flush = Some(f);
+                        None
+                    }
                 }
             }
         }
@@ -1076,14 +1082,10 @@ impl ScheduleStage {
         task_selection: &mut TaskSelection,
     ) -> Option<(TaskSource, TaskInQueue)> {
         let selected_heaviest_tasks = match task_selection {
-            TaskSelection::OnlyFromRunnable => (
-                runnable_queue.heaviest_entry_to_execute(),
-                None,
-            ),
-            TaskSelection::OnlyFromContended(_) => (
-                None,
-                Self::get_heaviest_from_contended(address_book),
-            ),
+            TaskSelection::OnlyFromRunnable => (runnable_queue.heaviest_entry_to_execute(), None),
+            TaskSelection::OnlyFromContended(_) => {
+                (None, Self::get_heaviest_from_contended(address_book))
+            }
         };
 
         match selected_heaviest_tasks {
@@ -1773,7 +1775,8 @@ impl ScheduleStage {
             let (mut from_len, mut from_exec_len) = (0, 0);
 
             loop {
-                let runnable_finished = from_disconnected && channel_backed_runnable_queue.has_no_task_hint();
+                let runnable_finished =
+                    from_disconnected && channel_backed_runnable_queue.has_no_task_hint();
 
                 let mut selection = TaskSelection::OnlyFromContended(if runnable_finished {
                     usize::max_value()
@@ -1901,7 +1904,8 @@ impl ScheduleStage {
                     break;
                 } else {
                     if !empty_from_exec {
-                        let UnlockablePayload(mut processed_execution_environment, extra) = from_exec.recv().unwrap();
+                        let UnlockablePayload(mut processed_execution_environment, extra) =
+                            from_exec.recv().unwrap();
                         from_exec_len = from_exec_len.checked_sub(1).unwrap();
                         empty_from_exec = from_exec_len == 0;
                         executing_queue_count = executing_queue_count.checked_sub(1).unwrap();
@@ -1914,7 +1918,10 @@ impl ScheduleStage {
                             &mut provisioning_tracker_count,
                         );
                         to_next_stage
-                            .send(ExaminablePayload(Flushable::Payload((processed_execution_environment, extra))))
+                            .send(ExaminablePayload(Flushable::Payload((
+                                processed_execution_environment,
+                                extra,
+                            ))))
                             .unwrap();
                     }
                     if !empty_from {
@@ -1948,7 +1955,11 @@ impl ScheduleStage {
             assert!(!select_skipped || executing_queue_count > 0);
         }
         if let Some(checkpoint) = &maybe_checkpoint {
-            to_next_stage.send(ExaminablePayload(Flushable::Flush(std::sync::Arc::clone(checkpoint)))).unwrap();
+            to_next_stage
+                .send(ExaminablePayload(Flushable::Flush(std::sync::Arc::clone(
+                    checkpoint,
+                ))))
+                .unwrap();
         }
         drop(to_next_stage);
         drop(ee_sender);
@@ -2049,9 +2060,7 @@ impl<T> Checkpoint<T> {
             );
             let _ = *self
                 .1
-                .wait_while(g, |&mut (remaining_threads, _)| {
-                    remaining_threads > 0
-                })
+                .wait_while(g, |&mut (remaining_threads, _)| remaining_threads > 0)
                 .unwrap();
             trace!(
                 "Checkpoint::wait_for_restart: {} is started...",

From e5c7535035119ace4ac439c76c9d8e105273c84b Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 23 Sep 2022 16:20:34 +0900
Subject: [PATCH 3128/3199] save

---
 scheduler/Cargo.toml | 1 -
 1 file changed, 1 deletion(-)

diff --git a/scheduler/Cargo.toml b/scheduler/Cargo.toml
index 4b1e00b0c553ed..cd45643a28d800 100644
--- a/scheduler/Cargo.toml
+++ b/scheduler/Cargo.toml
@@ -12,7 +12,6 @@ publish = true
 
 [dependencies]
 crossbeam-channel = "0.5.5"
-crossbeam-skiplist = { path = "../../../crossbeam/crossbeam-skiplist" }
 dashmap = { version = "4.0.2" }
 log = "0.4.17"
 solana-sdk = { path = "../sdk", version = "=1.12.0" }

From 38880c186655190aafc8586611b96a9d29322677 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 23 Sep 2022 16:21:41 +0900
Subject: [PATCH 3129/3199] save

---
 Cargo.lock           | 41 +++++------------------------------------
 scheduler/src/lib.rs |  6 ++++++
 2 files changed, 11 insertions(+), 36 deletions(-)

diff --git a/Cargo.lock b/Cargo.lock
index 06f45095c2e548..f219e7f87a1e59 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -1007,7 +1007,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "c2dd04ddaf88237dc3b8d8f9a3c1004b506b54b3313403944054d23c0870c521"
 dependencies = [
  "cfg-if 1.0.0",
- "crossbeam-utils 0.8.8",
+ "crossbeam-utils",
 ]
 
 [[package]]
@@ -1017,8 +1017,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "6455c0ca19f0d2fbf751b908d5c55c1f5cbc65e03c4225427254b46890bdde1e"
 dependencies = [
  "cfg-if 1.0.0",
- "crossbeam-epoch 0.9.5",
- "crossbeam-utils 0.8.8",
+ "crossbeam-epoch",
+ "crossbeam-utils",
 ]
 
 [[package]]
@@ -1027,34 +1027,12 @@ version = "0.9.5"
 source = "git+https://github.com/solana-labs/crossbeam?rev=fd279d707025f0e60951e429bf778b4813d1b6bf#fd279d707025f0e60951e429bf778b4813d1b6bf"
 dependencies = [
  "cfg-if 1.0.0",
- "crossbeam-utils 0.8.8",
+ "crossbeam-utils",
  "lazy_static",
  "memoffset",
  "scopeguard",
 ]
 
-[[package]]
-name = "crossbeam-epoch"
-version = "0.9.10"
-dependencies = [
- "autocfg",
- "cfg-if 1.0.0",
- "crossbeam-utils 0.8.11",
- "memoffset",
- "once_cell",
- "scopeguard",
-]
-
-[[package]]
-name = "crossbeam-skiplist"
-version = "0.0.0"
-dependencies = [
- "cfg-if 1.0.0",
- "crossbeam-epoch 0.9.10",
- "crossbeam-utils 0.8.11",
- "scopeguard",
-]
-
 [[package]]
 name = "crossbeam-utils"
 version = "0.8.8"
@@ -1065,14 +1043,6 @@ dependencies = [
  "lazy_static",
 ]
 
-[[package]]
-name = "crossbeam-utils"
-version = "0.8.11"
-dependencies = [
- "cfg-if 1.0.0",
- "once_cell",
-]
-
 [[package]]
 name = "crunchy"
 version = "0.2.2"
@@ -3838,7 +3808,7 @@ checksum = "9f51245e1e62e1f1629cbfec37b5793bbabcaeb90f30e94d2ba03564687353e4"
 dependencies = [
  "crossbeam-channel",
  "crossbeam-deque",
- "crossbeam-utils 0.8.8",
+ "crossbeam-utils",
  "num_cpus",
 ]
 
@@ -6207,7 +6177,6 @@ name = "solana-scheduler"
 version = "1.12.0"
 dependencies = [
  "crossbeam-channel",
- "crossbeam-skiplist",
  "dashmap 4.0.2",
  "log",
  "rand 0.7.3",
diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 2b5933c096f9a8..a7295c9b4f7aa6 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -196,19 +196,24 @@ pub enum RequestedUsage {
     Writable,
 }
 
+
+
 //type TaskIds = SkipListTaskIds;
 type TaskIds = BTreeMapTaskIds;
 
+/*
 #[derive(Debug, Default)]
 pub struct SkipListTaskIds {
     task_ids: crossbeam_skiplist::SkipMap<UniqueWeight, TaskInQueue>,
 }
+*/
 
 #[derive(Debug, Default)]
 pub struct BTreeMapTaskIds {
     task_ids: std::collections::BTreeMap<UniqueWeight, TaskInQueue>,
 }
 
+/*
 impl SkipListTaskIds {
     #[inline(never)]
     pub fn insert_task(&self, u: TaskId, task: TaskInQueue) {
@@ -271,6 +276,7 @@ impl SkipListTaskIds {
             .flatten()
     }
 }
+*/
 
 impl BTreeMapTaskIds {
     #[inline(never)]

From bb72ba7e0822af1edd2a8510cc2e136477eb134f Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Fri, 23 Sep 2022 16:23:19 +0900
Subject: [PATCH 3130/3199] save

---
 runtime/src/bank.rs  | 4 ++--
 scheduler/src/lib.rs | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 5133e178eb9e52..1a02f2ea484b17 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1317,7 +1317,7 @@ impl Scheduler<ExecuteTimings> {
         let bank = Arc::new(std::sync::RwLock::new(None::<std::sync::Weak<Bank>>));
 
         let executing_thread_count = std::env::var("EXECUTING_THREAD_COUNT")
-            .unwrap_or(format!("{}", 1))
+            .unwrap_or(format!("{}", 8))
             .parse::<usize>()
             .unwrap();
 
@@ -1485,7 +1485,7 @@ impl Scheduler<ExecuteTimings> {
                 }
 
                 let max_executing_queue_count = std::env::var("MAX_EXECUTING_QUEUE_COUNT")
-                    .unwrap_or(format!("{}", 1))
+                    .unwrap_or(format!("{}", 9))
                     .parse::<usize>()
                     .unwrap();
 
diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index a7295c9b4f7aa6..e9638e2bf54a35 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1670,7 +1670,7 @@ impl ScheduleStage {
         let (task_sender, task_receiver) =
             crossbeam_channel::unbounded::<(TaskInQueue, Vec<LockAttempt>)>();
         let indexer_count = std::env::var("INDEXER_COUNT")
-            .unwrap_or(format!("{}", 4))
+            .unwrap_or(format!("{}", 0))
             .parse::<usize>()
             .unwrap();
         let indexer_handles = (0..indexer_count)

From 538386b0d290ca6040139019fc43aece93bbba88 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 29 Sep 2022 12:40:39 +0900
Subject: [PATCH 3131/3199] Follow new naming

---
 runtime/src/bank.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 1a02f2ea484b17..90f5b0016917dc 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1430,7 +1430,7 @@ impl Scheduler<ExecuteTimings> {
 
                                 datapoint_info_at!(
                                     ee.finish_time.unwrap(),
-                                    "individual_tx_stats",
+                                    "transaction_timings",
                                     ("slot", ee.slot, i64),
                                     ("index", ee.transaction_index, i64),
                                     ("thread", format!("solScExLane{:02}", ee.thx), String),

From 593c1b3822bd3c8747a40896d87208c9754645dc Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 29 Sep 2022 12:57:16 +0900
Subject: [PATCH 3132/3199] save

---
 Cargo.toml           | 1 +
 runtime/src/bank.rs  | 4 +++-
 scheduler/src/lib.rs | 4 ++--
 3 files changed, 6 insertions(+), 3 deletions(-)

diff --git a/Cargo.toml b/Cargo.toml
index 2712d107ac27bb..c0d5d07c388068 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -106,3 +106,4 @@ resolver = "2"
 # for details, see https://github.com/solana-labs/crossbeam/commit/fd279d707025f0e60951e429bf778b4813d1b6bf
 [patch.crates-io]
 crossbeam-epoch = { git = "https://github.com/solana-labs/crossbeam", rev = "fd279d707025f0e60951e429bf778b4813d1b6bf" }
+crossbeam-channel = { git = "https://github.com/solana-labs/crossbeam", rev = "a8ef2a217c1ee3b54b57cc9117dceb8b64890a4b" }
diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 90f5b0016917dc..45ed9ad91c8145 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1420,7 +1420,8 @@ impl Scheduler<ExecuteTimings> {
 
                 let mut cumulative_timings = ExecuteTimings::default();
 
-                while let Ok(r) = retired_ee_receiver.recv()
+                loop {
+                while let Ok(r) = retired_ee_receiver.recv_timeout(std::time::Duration::from_millis(20))
                 {
                     match r {
                         solana_scheduler::ExaminablePayload(solana_scheduler::Flushable::Payload((mut ee, timings))) => {
@@ -1465,6 +1466,7 @@ impl Scheduler<ExecuteTimings> {
                         },
                     }
                 }
+                }
                 todo!();
 
                 Ok((started.0.elapsed(), started.1.elapsed()))
diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index e9638e2bf54a35..04395ac61eca9c 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1720,7 +1720,7 @@ impl ScheduleStage {
                            executing_queue_count = executing_queue_count.checked_sub(1).unwrap();
                            processed_count = processed_count.checked_add(1).unwrap();
                            Self::commit_processed_execution(ast, &mut processed_execution_environment, address_book, &mut commit_clock, &mut provisioning_tracker_count);
-                           to_next_stage.send(ExaminablePayload(Flushable::Payload((processed_execution_environment, extra)))).unwrap();
+                           to_next_stage.send_buffered(ExaminablePayload(Flushable::Payload((processed_execution_environment, extra)))).unwrap();
                        } else {
                            assert_eq!(from_exec.len(), 0);
                            from_exec_disconnected = true;
@@ -1924,7 +1924,7 @@ impl ScheduleStage {
                             &mut provisioning_tracker_count,
                         );
                         to_next_stage
-                            .send(ExaminablePayload(Flushable::Payload((
+                            .send_buffered(ExaminablePayload(Flushable::Payload((
                                 processed_execution_environment,
                                 extra,
                             ))))

From 6ca5e2a7fa5e0842719231dcc5851bb0c6caaeb5 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Thu, 29 Sep 2022 12:58:00 +0900
Subject: [PATCH 3133/3199] save

---
 Cargo.lock           | 20 ++++++++++++++------
 scheduler/src/lib.rs |  1 +
 2 files changed, 15 insertions(+), 6 deletions(-)

diff --git a/Cargo.lock b/Cargo.lock
index f219e7f87a1e59..5e2d6216c21ff6 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -1003,11 +1003,10 @@ dependencies = [
 [[package]]
 name = "crossbeam-channel"
 version = "0.5.6"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c2dd04ddaf88237dc3b8d8f9a3c1004b506b54b3313403944054d23c0870c521"
+source = "git+https://github.com/solana-labs/crossbeam?rev=a8ef2a217c1ee3b54b57cc9117dceb8b64890a4b#a8ef2a217c1ee3b54b57cc9117dceb8b64890a4b"
 dependencies = [
  "cfg-if 1.0.0",
- "crossbeam-utils",
+ "crossbeam-utils 0.8.11",
 ]
 
 [[package]]
@@ -1018,7 +1017,7 @@ checksum = "6455c0ca19f0d2fbf751b908d5c55c1f5cbc65e03c4225427254b46890bdde1e"
 dependencies = [
  "cfg-if 1.0.0",
  "crossbeam-epoch",
- "crossbeam-utils",
+ "crossbeam-utils 0.8.8",
 ]
 
 [[package]]
@@ -1027,7 +1026,7 @@ version = "0.9.5"
 source = "git+https://github.com/solana-labs/crossbeam?rev=fd279d707025f0e60951e429bf778b4813d1b6bf#fd279d707025f0e60951e429bf778b4813d1b6bf"
 dependencies = [
  "cfg-if 1.0.0",
- "crossbeam-utils",
+ "crossbeam-utils 0.8.8",
  "lazy_static",
  "memoffset",
  "scopeguard",
@@ -1043,6 +1042,15 @@ dependencies = [
  "lazy_static",
 ]
 
+[[package]]
+name = "crossbeam-utils"
+version = "0.8.11"
+source = "git+https://github.com/solana-labs/crossbeam?rev=a8ef2a217c1ee3b54b57cc9117dceb8b64890a4b#a8ef2a217c1ee3b54b57cc9117dceb8b64890a4b"
+dependencies = [
+ "cfg-if 1.0.0",
+ "once_cell",
+]
+
 [[package]]
 name = "crunchy"
 version = "0.2.2"
@@ -3808,7 +3816,7 @@ checksum = "9f51245e1e62e1f1629cbfec37b5793bbabcaeb90f30e94d2ba03564687353e4"
 dependencies = [
  "crossbeam-channel",
  "crossbeam-deque",
- "crossbeam-utils",
+ "crossbeam-utils 0.8.8",
  "num_cpus",
 ]
 
diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 04395ac61eca9c..1bf1343aaac06c 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1961,6 +1961,7 @@ impl ScheduleStage {
             assert!(!select_skipped || executing_queue_count > 0);
         }
         if let Some(checkpoint) = &maybe_checkpoint {
+            // wake up the receiver thread immediately by not using .send_buffered!
             to_next_stage
                 .send(ExaminablePayload(Flushable::Flush(std::sync::Arc::clone(
                     checkpoint,

From 776b8362f91c2457686f346e45fd4c0e02d2bdea Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 8 Oct 2022 14:02:42 +0900
Subject: [PATCH 3134/3199] save

---
 scheduler/src/lib.rs | 101 +++++++++++++++++++++++++------------------
 1 file changed, 60 insertions(+), 41 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 1bf1343aaac06c..99159f8e9c2546 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1714,49 +1714,68 @@ impl ScheduleStage {
             let mut select_skipped = false;
 
             if !from_disconnected || executing_queue_count >= 1 {
-                crossbeam_channel::select! {
-                   recv(from_exec) -> maybe_from_exec => {
-                       if let Ok(UnlockablePayload(mut processed_execution_environment, extra)) = maybe_from_exec {
-                           executing_queue_count = executing_queue_count.checked_sub(1).unwrap();
-                           processed_count = processed_count.checked_add(1).unwrap();
-                           Self::commit_processed_execution(ast, &mut processed_execution_environment, address_book, &mut commit_clock, &mut provisioning_tracker_count);
-                           to_next_stage.send_buffered(ExaminablePayload(Flushable::Payload((processed_execution_environment, extra)))).unwrap();
-                       } else {
-                           assert_eq!(from_exec.len(), 0);
-                           from_exec_disconnected = true;
-                           info!("flushing1..: {:?} {} {} {} {}", (from_disconnected, from_exec_disconnected), channel_backed_runnable_queue.task_count_hint(), contended_count, executing_queue_count, provisioning_tracker_count);
-                           if from_disconnected {
-                               break;
-                           }
-                       }
-                   }
-                   recv(from_prev) -> maybe_from => {
-                       match maybe_from {
-                           Ok(SchedulablePayload(Flushable::Payload(task))) => {
-                               if maybe_start_time.is_none() {
-                                    info!("schedule_once:initial id_{:016x}", random_id);
-                                   maybe_start_time = Some(std::time::Instant::now());
-                                   last_time = maybe_start_time.clone();
-                               }
-                               //Self::register_runnable_task(task, runnable_queue, &mut sequence_time);
-                               channel_backed_runnable_queue.buffer(task);
-                           },
-                           Ok(SchedulablePayload(Flushable::Flush(checkpoint))) => {
-                               assert_eq!(from_prev.len(), 0);
-                               assert!(!from_disconnected);
-                               from_disconnected = true;
-                               from_prev = never;
-                               trace!("flushing2..: {:?} {} {} {} {}", (from_disconnected, from_exec_disconnected), channel_backed_runnable_queue.task_count_hint(), contended_count, executing_queue_count, provisioning_tracker_count);
-                               assert!(maybe_checkpoint.is_none());
-                               maybe_checkpoint = Some(checkpoint);
+                if !from_disconnected {
+                    crossbeam_channel::select! {
+                        recv(from_exec) -> maybe_from_exec => {
+                            if let Ok(UnlockablePayload(mut processed_execution_environment, extra)) = maybe_from_exec {
+                                executing_queue_count = executing_queue_count.checked_sub(1).unwrap();
+                                processed_count = processed_count.checked_add(1).unwrap();
+                                Self::commit_processed_execution(ast, &mut processed_execution_environment, address_book, &mut commit_clock, &mut provisioning_tracker_count);
+                                to_next_stage.send_buffered(ExaminablePayload(Flushable::Payload((processed_execution_environment, extra)))).unwrap();
+                            } else {
+                                assert_eq!(from_exec.len(), 0);
+                                from_exec_disconnected = true;
+                                info!("flushing1..: {:?} {} {} {} {}", (from_disconnected, from_exec_disconnected), channel_backed_runnable_queue.task_count_hint(), contended_count, executing_queue_count, provisioning_tracker_count);
+                                if from_disconnected {
+                                    break;
+                                }
+                            }
+                        }
+                        recv(from_prev) -> maybe_from => {
+                            match maybe_from {
+                                Ok(SchedulablePayload(Flushable::Payload(task))) => {
+                                    if maybe_start_time.is_none() {
+                                         info!("schedule_once:initial id_{:016x}", random_id);
+                                        maybe_start_time = Some(std::time::Instant::now());
+                                        last_time = maybe_start_time.clone();
+                                    }
+                                    //Self::register_runnable_task(task, runnable_queue, &mut sequence_time);
+                                    channel_backed_runnable_queue.buffer(task);
+                                },
+                                Ok(SchedulablePayload(Flushable::Flush(checkpoint))) => {
+                                    assert_eq!(from_prev.len(), 0);
+                                    assert!(!from_disconnected);
+                                    from_disconnected = true;
+                                    from_prev = never;
+                                    trace!("flushing2..: {:?} {} {} {} {}", (from_disconnected, from_exec_disconnected), channel_backed_runnable_queue.task_count_hint(), contended_count, executing_queue_count, provisioning_tracker_count);
+                                    assert!(maybe_checkpoint.is_none());
+                                    maybe_checkpoint = Some(checkpoint);
+                                },
+                                Err(_) => {
+                                    assert_eq!(from_prev.len(), 0);
+                                    assert!(!from_disconnected);
+                                    assert!(maybe_checkpoint.is_none());
+                                    from_disconnected = true;
+                                    from_prev = never;
+                                    trace!("flushing2..: {:?} {} {} {} {}", (from_disconnected, from_exec_disconnected), channel_backed_runnable_queue.task_count_hint(), contended_count, executing_queue_count, provisioning_tracker_count);
+                                },
+                            }
+                        }
+                    }
+                } else {
+                    loop {
+                        match maybe_from_exec.try_recv() {
+                           Ok(UnlockablePayload(mut processed_execution_environment, extra)) => {
+                               executing_queue_count = executing_queue_count.checked_sub(1).unwrap();
+                               processed_count = processed_count.checked_add(1).unwrap();
+                               Self::commit_processed_execution(ast, &mut processed_execution_environment, address_book, &mut commit_clock, &mut provisioning_tracker_count);
+                               to_next_stage.send_buffered(ExaminablePayload(Flushable::Payload((processed_execution_environment, extra)))).unwrap();
                            },
                            Err(_) => {
-                               assert_eq!(from_prev.len(), 0);
-                               assert!(!from_disconnected);
-                               assert!(maybe_checkpoint.is_none());
-                               from_disconnected = true;
-                               from_prev = never;
-                               trace!("flushing2..: {:?} {} {} {} {}", (from_disconnected, from_exec_disconnected), channel_backed_runnable_queue.task_count_hint(), contended_count, executing_queue_count, provisioning_tracker_count);
+                               assert_eq!(from_exec.len(), 0);
+                               from_exec_disconnected = true;
+                               info!("flushing1..: {:?} {} {} {} {}", (from_disconnected, from_exec_disconnected), channel_backed_runnable_queue.task_count_hint(), contended_count, executing_queue_count, provisioning_tracker_count);
+                               break;
                            },
                        }
                    }

From 9762363a5b4da8c27a7f6453eacd4f37f3ea22e8 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 8 Oct 2022 14:03:18 +0900
Subject: [PATCH 3135/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 99159f8e9c2546..ed168771e5954c 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1764,7 +1764,7 @@ impl ScheduleStage {
                     }
                 } else {
                     loop {
-                        match maybe_from_exec.try_recv() {
+                        match from_exec.try_recv() {
                            Ok(UnlockablePayload(mut processed_execution_environment, extra)) => {
                                executing_queue_count = executing_queue_count.checked_sub(1).unwrap();
                                processed_count = processed_count.checked_add(1).unwrap();

From 8a2d4cd7f3fef3b0aeb86dfa4c14b3c2e81cbba6 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 8 Oct 2022 14:04:19 +0900
Subject: [PATCH 3136/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index ed168771e5954c..19f6760e765467 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1771,7 +1771,7 @@ impl ScheduleStage {
                                Self::commit_processed_execution(ast, &mut processed_execution_environment, address_book, &mut commit_clock, &mut provisioning_tracker_count);
                                to_next_stage.send_buffered(ExaminablePayload(Flushable::Payload((processed_execution_environment, extra)))).unwrap();
                            },
-                           Err(_) => {
+                           Err(TryRecvError::Disconnected) => {
                                assert_eq!(from_exec.len(), 0);
                                from_exec_disconnected = true;
                                info!("flushing1..: {:?} {} {} {} {}", (from_disconnected, from_exec_disconnected), channel_backed_runnable_queue.task_count_hint(), contended_count, executing_queue_count, provisioning_tracker_count);

From ace0dd1a9a5b01dc2650f4c213fbe9f2506b5169 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 8 Oct 2022 14:05:27 +0900
Subject: [PATCH 3137/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 19f6760e765467..9b3f291483e9b3 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1771,7 +1771,7 @@ impl ScheduleStage {
                                Self::commit_processed_execution(ast, &mut processed_execution_environment, address_book, &mut commit_clock, &mut provisioning_tracker_count);
                                to_next_stage.send_buffered(ExaminablePayload(Flushable::Payload((processed_execution_environment, extra)))).unwrap();
                            },
-                           Err(TryRecvError::Disconnected) => {
+                           Err(crossbeam_channel::TryRecvError::Disconnected) => {
                                assert_eq!(from_exec.len(), 0);
                                from_exec_disconnected = true;
                                info!("flushing1..: {:?} {} {} {} {}", (from_disconnected, from_exec_disconnected), channel_backed_runnable_queue.task_count_hint(), contended_count, executing_queue_count, provisioning_tracker_count);

From c384f7664b657d64c4f039d4bef2a0e0509737fa Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 8 Oct 2022 14:06:01 +0900
Subject: [PATCH 3138/3199] save

---
 scheduler/src/lib.rs | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 9b3f291483e9b3..93d8d9c1d027f3 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1771,6 +1771,9 @@ impl ScheduleStage {
                                Self::commit_processed_execution(ast, &mut processed_execution_environment, address_book, &mut commit_clock, &mut provisioning_tracker_count);
                                to_next_stage.send_buffered(ExaminablePayload(Flushable::Payload((processed_execution_environment, extra)))).unwrap();
                            },
+                           Err(crossbeam_channel::TryRecvError::Empty) => {
+                               continue;
+                           },
                            Err(crossbeam_channel::TryRecvError::Disconnected) => {
                                assert_eq!(from_exec.len(), 0);
                                from_exec_disconnected = true;

From e8631a4a9c8833ef638ce6c1bb43aac6d6af77b0 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 8 Oct 2022 14:07:06 +0900
Subject: [PATCH 3139/3199] save

---
 scheduler/src/lib.rs | 1 +
 1 file changed, 1 insertion(+)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 93d8d9c1d027f3..e1ca0a955f7db8 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1770,6 +1770,7 @@ impl ScheduleStage {
                                processed_count = processed_count.checked_add(1).unwrap();
                                Self::commit_processed_execution(ast, &mut processed_execution_environment, address_book, &mut commit_clock, &mut provisioning_tracker_count);
                                to_next_stage.send_buffered(ExaminablePayload(Flushable::Payload((processed_execution_environment, extra)))).unwrap();
+                               break;
                            },
                            Err(crossbeam_channel::TryRecvError::Empty) => {
                                continue;

From b83df9660be1cacfb391be994adcb355adf99699 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 8 Oct 2022 14:07:25 +0900
Subject: [PATCH 3140/3199] save

---
 scheduler/src/lib.rs | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index e1ca0a955f7db8..37f8a8f7f7a444 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1765,6 +1765,9 @@ impl ScheduleStage {
                 } else {
                     loop {
                         match from_exec.try_recv() {
+                           Err(crossbeam_channel::TryRecvError::Empty) => {
+                               continue;
+                           },
                            Ok(UnlockablePayload(mut processed_execution_environment, extra)) => {
                                executing_queue_count = executing_queue_count.checked_sub(1).unwrap();
                                processed_count = processed_count.checked_add(1).unwrap();
@@ -1772,9 +1775,6 @@ impl ScheduleStage {
                                to_next_stage.send_buffered(ExaminablePayload(Flushable::Payload((processed_execution_environment, extra)))).unwrap();
                                break;
                            },
-                           Err(crossbeam_channel::TryRecvError::Empty) => {
-                               continue;
-                           },
                            Err(crossbeam_channel::TryRecvError::Disconnected) => {
                                assert_eq!(from_exec.len(), 0);
                                from_exec_disconnected = true;

From 618373b37787e182bb73ab2b7f5dc98feaef10ee Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 8 Oct 2022 14:11:44 +0900
Subject: [PATCH 3141/3199] save

---
 scheduler/src/lib.rs | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 37f8a8f7f7a444..fb77a353f18b13 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -2117,3 +2117,8 @@ pub enum Flushable<T, C> {
     Payload(T),
     Flush(std::sync::Arc<Checkpoint<C>>),
 }
+
+pub enum SpinWaitable<T> {
+    Payload(T),
+    Spin,
+}

From 65bb5cf424ce48b8326bca6654f97c3d5a4f194c Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 8 Oct 2022 14:12:45 +0900
Subject: [PATCH 3142/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index fb77a353f18b13..b8448c87ccb327 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -2050,7 +2050,7 @@ impl ScheduleStage {
 }
 
 pub struct SchedulablePayload<C>(pub Flushable<TaskInQueue, C>);
-pub struct ExecutablePayload(pub Box<ExecutionEnvironment>);
+pub struct ExecutablePayload(pub SpinWaitable<Box<ExecutionEnvironment>>);
 pub struct UnlockablePayload<T>(pub Box<ExecutionEnvironment>, pub T);
 pub struct ExaminablePayload<T, C>(pub Flushable<(Box<ExecutionEnvironment>, T), C>);
 

From cc80c93f0fa954e6dec5802cfa69a4f689c59d2f Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 8 Oct 2022 14:13:04 +0900
Subject: [PATCH 3143/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index b8448c87ccb327..48aeda6fa9b880 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1880,7 +1880,7 @@ impl ScheduleStage {
                     );
                     if let Some(ee) = maybe_ee {
                         executing_queue_count = executing_queue_count.checked_add(1).unwrap();
-                        to_execute_substage.send(ExecutablePayload(ee)).unwrap();
+                        to_execute_substage.send(ExecutablePayload(SpinWaitable::Payload(ee))).unwrap();
                     }
                     debug!("schedule_once id_{:016x} [R] ch(prev: {}, exec: {}+{}|{}), r: {}, u/c: {}/{}, (imm+provi)/max: ({}+{})/{} s: {} l(s+f): {}+{}", random_id, (if from_disconnected { "-".to_string() } else { format!("{}", from_prev.len()) }), to_high_execute_substage.map(|t| format!("{}", t.len())).unwrap_or("-".into()), to_execute_substage.len(), from_exec.len(), channel_backed_runnable_queue.task_count_hint(), address_book.uncontended_task_ids.len(), contended_count, executing_queue_count, provisioning_tracker_count, max_executing_queue_count, address_book.stuck_tasks.len(), processed_count, failed_lock_count);
 

From 0fe7bb06e8e9d7056b8471ec5a93dd6ed1426966 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 8 Oct 2022 14:13:29 +0900
Subject: [PATCH 3144/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 48aeda6fa9b880..60dfa18d265f14 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1833,7 +1833,7 @@ impl ScheduleStage {
                         executing_queue_count = executing_queue_count.checked_add(1).unwrap();
                         to_high_execute_substage
                             .unwrap_or(to_execute_substage)
-                            .send(ExecutablePayload(ee))
+                            .send(ExecutablePayload(SpinWaitable::Payload(ee)))
                             .unwrap();
                     }
                     debug!("schedule_once id_{:016x} [C] ch(prev: {}, exec: {}+{}|{}), r: {}, u/c: {}/{}, (imm+provi)/max: ({}+{})/{} s: {} l(s+f): {}+{}", random_id, (if from_disconnected { "-".to_string() } else { format!("{}", from_prev.len()) }), to_high_execute_substage.map(|t| format!("{}", t.len())).unwrap_or("-".into()), to_execute_substage.len(), from_exec.len(), channel_backed_runnable_queue.task_count_hint(), address_book.uncontended_task_ids.len(), contended_count, executing_queue_count, provisioning_tracker_count, max_executing_queue_count, address_book.stuck_tasks.len(), processed_count, failed_lock_count);

From cba84598b6f426915b9b10b97e47af78bc3d5275 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 8 Oct 2022 14:14:27 +0900
Subject: [PATCH 3145/3199] save

---
 runtime/src/bank.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 45ed9ad91c8145..4fb514295bbcf3 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1335,7 +1335,7 @@ impl Scheduler<ExecuteTimings> {
                 thread_priority::set_current_thread_priority(thread_priority::ThreadPriority::Max).unwrap();
             }
 
-            while let Ok(solana_scheduler::ExecutablePayload(mut ee)) = (if thx >= executing_thread_count { scheduled_high_ee_receiver.recv() } else { scheduled_ee_receiver.recv()}) {
+            while let Ok(solana_scheduler::ExecutablePayload(solana_scheduler::SpinWaitable::Payload(mut ee))) = (if thx >= executing_thread_count { scheduled_high_ee_receiver.recv() } else { scheduled_ee_receiver.recv()}) {
                 let (mut wall_time, cpu_time) = (Measure::start("process_message_time"), cpu_time::ThreadTime::now());
 
                 let current_execute_clock = ee.task.execute_time();

From b981bdaae707de81e58ee99bccde0b5396df0ee0 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 8 Oct 2022 14:15:08 +0900
Subject: [PATCH 3146/3199] save

---
 ledger-tool/src/main.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ledger-tool/src/main.rs b/ledger-tool/src/main.rs
index 57bda5bf6d2088..3b5eb68b486a4c 100644
--- a/ledger-tool/src/main.rs
+++ b/ledger-tool/src/main.rs
@@ -316,7 +316,7 @@ fn output_slot(
                     let send_metrics = std::env::var("SEND_METRICS").is_ok();
 
                     for step in 0.. {
-                        let mut ee = pre_execute_env_receiver.recv().unwrap().0;
+                        let mut ee = pre_execute_env_receiver.recv().unwrap().0.0;
                         if step % 1966 == 0 {
                             error!("executing!: {} {}", step, pre_execute_env_receiver.len());
                         }

From 68e440f732ab92dfbd29fcd310fff50a7ada22e3 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 8 Oct 2022 14:16:03 +0900
Subject: [PATCH 3147/3199] save

---
 ledger-tool/src/main.rs | 73 +++++++++++++++++++++--------------------
 1 file changed, 37 insertions(+), 36 deletions(-)

diff --git a/ledger-tool/src/main.rs b/ledger-tool/src/main.rs
index 3b5eb68b486a4c..a1952de504531e 100644
--- a/ledger-tool/src/main.rs
+++ b/ledger-tool/src/main.rs
@@ -316,44 +316,45 @@ fn output_slot(
                     let send_metrics = std::env::var("SEND_METRICS").is_ok();
 
                     for step in 0.. {
-                        let mut ee = pre_execute_env_receiver.recv().unwrap().0.0;
-                        if step % 1966 == 0 {
-                            error!("executing!: {} {}", step, pre_execute_env_receiver.len());
-                        }
+                        if let solana_scheduler::SpinWaitable::Payload(mut ee) = pre_execute_env_receiver.recv().unwrap().0 {
+                            if step % 1966 == 0 {
+                                error!("executing!: {} {}", step, pre_execute_env_receiver.len());
+                            }
 
-                        if send_metrics {
-                            let mut process_message_time = Measure::start("process_message_time");
-                            let sig = ee.task.tx.0.signature().to_string();
-                            trace!("execute substage: #{} {:#?}", step, &sig);
-                            std::thread::sleep(std::time::Duration::from_micros(
-                                ee.cu.try_into().unwrap(),
-                            ));
-
-                            process_message_time.stop();
-                            let duration_with_overhead = process_message_time.as_us();
-
-                            datapoint_info!(
-                                "individual_tx_stats",
-                                ("slot", 33333, i64),
-                                ("thread", current_thread_name, String),
-                                ("signature", &sig, String),
-                                ("account_locks_in_json", "{}", String),
-                                ("status", "Ok", String),
-                                ("duration", duration_with_overhead, i64),
-                                ("compute_units", ee.cu, i64),
-                            );
-                        }
+                            if send_metrics {
+                                let mut process_message_time = Measure::start("process_message_time");
+                                let sig = ee.task.tx.0.signature().to_string();
+                                trace!("execute substage: #{} {:#?}", step, &sig);
+                                std::thread::sleep(std::time::Duration::from_micros(
+                                    ee.cu.try_into().unwrap(),
+                                ));
+
+                                process_message_time.stop();
+                                let duration_with_overhead = process_message_time.as_us();
+
+                                datapoint_info!(
+                                    "individual_tx_stats",
+                                    ("slot", 33333, i64),
+                                    ("thread", current_thread_name, String),
+                                    ("signature", &sig, String),
+                                    ("account_locks_in_json", "{}", String),
+                                    ("status", "Ok", String),
+                                    ("duration", duration_with_overhead, i64),
+                                    ("compute_units", ee.cu, i64),
+                                );
+                            }
 
-                        /*
-                        muxed_sender
-                            .send(solana_scheduler::Multiplexed::FromExecute(ee))
-                            .unwrap();
-                        */
-                        // ee.reindex_with_address_book();
-                        todo!("contended_write_task_count");
-                        post_execute_env_sender
-                            .send(solana_scheduler::UnlockablePayload(ee, ()))
-                            .unwrap();
+                            /*
+                            muxed_sender
+                                .send(solana_scheduler::Multiplexed::FromExecute(ee))
+                                .unwrap();
+                            */
+                            // ee.reindex_with_address_book();
+                            todo!("contended_write_task_count");
+                            post_execute_env_sender
+                                .send(solana_scheduler::UnlockablePayload(ee, ()))
+                                .unwrap();
+                        }
                     }
                 })
                 .unwrap();

From 3b88454c61c49e31d220b9c4f6b451933a65103d Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 8 Oct 2022 14:19:22 +0900
Subject: [PATCH 3148/3199] save

---
 runtime/src/bank.rs | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 4fb514295bbcf3..00dbfff2935b92 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1335,7 +1335,10 @@ impl Scheduler<ExecuteTimings> {
                 thread_priority::set_current_thread_priority(thread_priority::ThreadPriority::Max).unwrap();
             }
 
-            while let Ok(solana_scheduler::ExecutablePayload(solana_scheduler::SpinWaitable::Payload(mut ee))) = (if thx >= executing_thread_count { scheduled_high_ee_receiver.recv() } else { scheduled_ee_receiver.recv()}) {
+            loop {
+            let received = (if thx >= executing_thread_count { scheduled_high_ee_receiver.recv() } else { scheduled_ee_receiver.recv()});
+            match recorded {
+            Ok(solana_scheduler::ExecutablePayload(solana_scheduler::SpinWaitable::Payload(mut ee))) => {
                 let (mut wall_time, cpu_time) = (Measure::start("process_message_time"), cpu_time::ThreadTime::now());
 
                 let current_execute_clock = ee.task.execute_time();
@@ -1398,6 +1401,8 @@ impl Scheduler<ExecuteTimings> {
 
                 //ee.reindex_with_address_book();
                 processed_ee_sender.send(solana_scheduler::UnlockablePayload(ee, timings)).unwrap();
+            },
+            }
             }
             todo!();
 

From ac19a68317af86a17bc52dd6e7e885f635ff6e24 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 8 Oct 2022 14:19:33 +0900
Subject: [PATCH 3149/3199] save

---
 runtime/src/bank.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 00dbfff2935b92..a6b296fab64276 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1337,7 +1337,7 @@ impl Scheduler<ExecuteTimings> {
 
             loop {
             let received = (if thx >= executing_thread_count { scheduled_high_ee_receiver.recv() } else { scheduled_ee_receiver.recv()});
-            match recorded {
+            match received {
             Ok(solana_scheduler::ExecutablePayload(solana_scheduler::SpinWaitable::Payload(mut ee))) => {
                 let (mut wall_time, cpu_time) = (Measure::start("process_message_time"), cpu_time::ThreadTime::now());
 

From edeba45dffc967ea7277c571f16c7a24872cf714 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 8 Oct 2022 14:20:06 +0900
Subject: [PATCH 3150/3199] save

---
 runtime/src/bank.rs | 1 +
 1 file changed, 1 insertion(+)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index a6b296fab64276..6d24d0d4615f14 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1402,6 +1402,7 @@ impl Scheduler<ExecuteTimings> {
                 //ee.reindex_with_address_book();
                 processed_ee_sender.send(solana_scheduler::UnlockablePayload(ee, timings)).unwrap();
             },
+            Err(_) => todo!(),
             }
             }
             todo!();

From 3934f48a20965c0e56a1175bef901bd1a9d1752d Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 8 Oct 2022 14:20:51 +0900
Subject: [PATCH 3151/3199] save

---
 runtime/src/bank.rs | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 6d24d0d4615f14..5ed45803d4791e 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1338,6 +1338,8 @@ impl Scheduler<ExecuteTimings> {
             loop {
             let received = (if thx >= executing_thread_count { scheduled_high_ee_receiver.recv() } else { scheduled_ee_receiver.recv()});
             match received {
+            Ok(solana_scheduler::ExecutablePayload(solana_scheduler::SpinWaitable::Spin)) => {
+            },
             Ok(solana_scheduler::ExecutablePayload(solana_scheduler::SpinWaitable::Payload(mut ee))) => {
                 let (mut wall_time, cpu_time) = (Measure::start("process_message_time"), cpu_time::ThreadTime::now());
 

From cc434a7ca2e6c1a3cd2aef4bf965deb85e64e105 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 8 Oct 2022 14:21:37 +0900
Subject: [PATCH 3152/3199] save

---
 runtime/src/bank.rs | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 5ed45803d4791e..807a448a6ea4bb 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1335,8 +1335,10 @@ impl Scheduler<ExecuteTimings> {
                 thread_priority::set_current_thread_priority(thread_priority::ThreadPriority::Max).unwrap();
             }
 
+            let r = (if thx >= executing_thread_count { scheduled_high_ee_receiver } else { scheduled_ee_receiver});
+
             loop {
-            let received = (if thx >= executing_thread_count { scheduled_high_ee_receiver.recv() } else { scheduled_ee_receiver.recv()});
+            let received = r.recv();
             match received {
             Ok(solana_scheduler::ExecutablePayload(solana_scheduler::SpinWaitable::Spin)) => {
             },

From 31b5a2116d10a64bc45b3133a609804f4f4ddb8d Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 8 Oct 2022 14:24:37 +0900
Subject: [PATCH 3153/3199] save

---
 runtime/src/bank.rs | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 807a448a6ea4bb..5598691e123db9 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1338,11 +1338,19 @@ impl Scheduler<ExecuteTimings> {
             let r = (if thx >= executing_thread_count { scheduled_high_ee_receiver } else { scheduled_ee_receiver});
 
             loop {
+            let mut maybe_ee = None;
+
             let received = r.recv();
             match received {
             Ok(solana_scheduler::ExecutablePayload(solana_scheduler::SpinWaitable::Spin)) => {
             },
             Ok(solana_scheduler::ExecutablePayload(solana_scheduler::SpinWaitable::Payload(mut ee))) => {
+                maybe_ee = Some(ee);
+            },
+            Err(_) => todo!(),
+            }
+
+            if let Some(ee) = maybe_ee {
                 let (mut wall_time, cpu_time) = (Measure::start("process_message_time"), cpu_time::ThreadTime::now());
 
                 let current_execute_clock = ee.task.execute_time();
@@ -1405,8 +1413,6 @@ impl Scheduler<ExecuteTimings> {
 
                 //ee.reindex_with_address_book();
                 processed_ee_sender.send(solana_scheduler::UnlockablePayload(ee, timings)).unwrap();
-            },
-            Err(_) => todo!(),
             }
             }
             todo!();

From c195b4bd06d6c7b11369ffe38bd8f6b2d73e3087 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 8 Oct 2022 14:24:53 +0900
Subject: [PATCH 3154/3199] save

---
 runtime/src/bank.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 5598691e123db9..61e439008607db 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1350,7 +1350,7 @@ impl Scheduler<ExecuteTimings> {
             Err(_) => todo!(),
             }
 
-            if let Some(ee) = maybe_ee {
+            if let Some(mut ee) = maybe_ee {
                 let (mut wall_time, cpu_time) = (Measure::start("process_message_time"), cpu_time::ThreadTime::now());
 
                 let current_execute_clock = ee.task.execute_time();

From 324633f9ee15aad6a065c5690c8a1155194e482c Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 8 Oct 2022 14:27:54 +0900
Subject: [PATCH 3155/3199] save

---
 runtime/src/bank.rs | 17 +++++++++++++++++
 1 file changed, 17 insertions(+)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 61e439008607db..c827ef5dd171cd 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1343,6 +1343,23 @@ impl Scheduler<ExecuteTimings> {
             let received = r.recv();
             match received {
             Ok(solana_scheduler::ExecutablePayload(solana_scheduler::SpinWaitable::Spin)) => {
+                loop {
+                    match r.try_recv() {
+                        Err(crossbeam_channel::TryRecvError::Empty) => {
+                            continue;
+                        },
+                        Ok(solana_scheduler::ExecutablePayload(solana_scheduler::SpinWaitable::Payload(ee))) => {
+                            maybe_ee = Some(ee);
+                            break;
+                        }
+                        Ok(solana_scheduler::ExecutablePayload(solana_scheduler::SpinWaitable::Spin)) => {
+                            unreachable!();
+                        }
+                        Err(crossbeam_channel::TryRecvError::Disconnected) => {
+                            continue;
+                        },
+                    }
+                }
             },
             Ok(solana_scheduler::ExecutablePayload(solana_scheduler::SpinWaitable::Payload(mut ee))) => {
                 maybe_ee = Some(ee);

From 70779309a05be44d4dcdda67c46ba71b5358a883 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 8 Oct 2022 14:32:36 +0900
Subject: [PATCH 3156/3199] save

---
 runtime/src/bank.rs  | 46 ++++++++++++++++++++++++--------------------
 scheduler/src/lib.rs |  1 +
 2 files changed, 26 insertions(+), 21 deletions(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index c827ef5dd171cd..85b99868e2adc6 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1342,29 +1342,31 @@ impl Scheduler<ExecuteTimings> {
 
             let received = r.recv();
             match received {
-            Ok(solana_scheduler::ExecutablePayload(solana_scheduler::SpinWaitable::Spin)) => {
-                loop {
-                    match r.try_recv() {
-                        Err(crossbeam_channel::TryRecvError::Empty) => {
-                            continue;
-                        },
-                        Ok(solana_scheduler::ExecutablePayload(solana_scheduler::SpinWaitable::Payload(ee))) => {
-                            maybe_ee = Some(ee);
-                            break;
-                        }
-                        Ok(solana_scheduler::ExecutablePayload(solana_scheduler::SpinWaitable::Spin)) => {
-                            unreachable!();
+                Ok(a = solana_scheduler::ExecutablePayload(solana_scheduler::SpinWaitable::Spin)) => {
+                    loop {
+                        match r.try_recv() {
+                            Err(crossbeam_channel::TryRecvError::Empty) => {
+                                // let's spin
+                                continue;
+                            },
+                            Ok(solana_scheduler::ExecutablePayload(solana_scheduler::SpinWaitable::Payload(ee))) => {
+                                s.send(spin).unwrap(); 
+                                maybe_ee = Some(ee);
+                                break;
+                            }
+                            Ok(solana_scheduler::ExecutablePayload(solana_scheduler::SpinWaitable::Spin)) => {
+                                unreachable!();
+                            }
+                            Err(crossbeam_channel::TryRecvError::Disconnected) => {
+                                continue;
+                            },
                         }
-                        Err(crossbeam_channel::TryRecvError::Disconnected) => {
-                            continue;
-                        },
                     }
-                }
-            },
-            Ok(solana_scheduler::ExecutablePayload(solana_scheduler::SpinWaitable::Payload(mut ee))) => {
-                maybe_ee = Some(ee);
-            },
-            Err(_) => todo!(),
+                },
+                Ok(solana_scheduler::ExecutablePayload(solana_scheduler::SpinWaitable::Payload(mut ee))) => {
+                    maybe_ee = Some(ee);
+                },
+                Err(_) => todo!(),
             }
 
             if let Some(mut ee) = maybe_ee {
@@ -1430,6 +1432,8 @@ impl Scheduler<ExecuteTimings> {
 
                 //ee.reindex_with_address_book();
                 processed_ee_sender.send(solana_scheduler::UnlockablePayload(ee, timings)).unwrap();
+            } else {
+                unreachable!();
             }
             }
             todo!();
diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 60dfa18d265f14..f55153c45d843c 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1766,6 +1766,7 @@ impl ScheduleStage {
                     loop {
                         match from_exec.try_recv() {
                            Err(crossbeam_channel::TryRecvError::Empty) => {
+                               // let's spin
                                continue;
                            },
                            Ok(UnlockablePayload(mut processed_execution_environment, extra)) => {

From 978aa99f442751de0c7914486542f06811436542 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 8 Oct 2022 14:33:10 +0900
Subject: [PATCH 3157/3199] save

---
 runtime/src/bank.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 85b99868e2adc6..0de35f0e61e854 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1342,7 +1342,7 @@ impl Scheduler<ExecuteTimings> {
 
             let received = r.recv();
             match received {
-                Ok(a = solana_scheduler::ExecutablePayload(solana_scheduler::SpinWaitable::Spin)) => {
+                Ok(solana_scheduler::ExecutablePayload(solana_scheduler::SpinWaitable::Spin)) => {
                     loop {
                         match r.try_recv() {
                             Err(crossbeam_channel::TryRecvError::Empty) => {
@@ -1350,7 +1350,7 @@ impl Scheduler<ExecuteTimings> {
                                 continue;
                             },
                             Ok(solana_scheduler::ExecutablePayload(solana_scheduler::SpinWaitable::Payload(ee))) => {
-                                s.send(spin).unwrap(); 
+                                s.send(solana_scheduler::ExecutablePayload(solana_scheduler::SpinWaitable::Spin)).unwrap(); 
                                 maybe_ee = Some(ee);
                                 break;
                             }

From f620e309b09559bb28cde5daa98404428d55a548 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 8 Oct 2022 14:34:45 +0900
Subject: [PATCH 3158/3199] save

---
 runtime/src/bank.rs | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 0de35f0e61e854..35e5c95faa5c94 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1327,6 +1327,7 @@ impl Scheduler<ExecuteTimings> {
 
         let executing_thread_handles = (0..(executing_thread_count * 2)).map(|thx| {
             let (scheduled_ee_receiver, scheduled_high_ee_receiver, processed_ee_sender) = (scheduled_ee_receiver.clone(), scheduled_high_ee_receiver.clone(), processed_ee_sender.clone());
+            let (scheduled_ee_sender, scheduled_high_ee_sender) = (scheduled_ee_sender.clone(), scheduled_high_ee_sender.clone());
             let bank = bank.clone();
 
             std::thread::Builder::new().name(format!("solScExLane{:02}", thx)).spawn(move || {
@@ -1336,6 +1337,7 @@ impl Scheduler<ExecuteTimings> {
             }
 
             let r = (if thx >= executing_thread_count { scheduled_high_ee_receiver } else { scheduled_ee_receiver});
+            let s = (if thx >= executing_thread_count { scheduled_high_ee_sener } else { scheduled_ee_sender});
 
             loop {
             let mut maybe_ee = None;

From 59bcc3d3205b1b1bae45c05db8bc59d5ee6891af Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 8 Oct 2022 14:34:59 +0900
Subject: [PATCH 3159/3199] save

---
 runtime/src/bank.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 35e5c95faa5c94..622b1bc652faad 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1337,7 +1337,7 @@ impl Scheduler<ExecuteTimings> {
             }
 
             let r = (if thx >= executing_thread_count { scheduled_high_ee_receiver } else { scheduled_ee_receiver});
-            let s = (if thx >= executing_thread_count { scheduled_high_ee_sener } else { scheduled_ee_sender});
+            let s = (if thx >= executing_thread_count { scheduled_high_ee_sender } else { scheduled_ee_sender});
 
             loop {
             let mut maybe_ee = None;

From 6853dff6e407c0df7d830b7eac133ab9571e684b Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 8 Oct 2022 14:39:59 +0900
Subject: [PATCH 3160/3199] save

---
 runtime/src/bank.rs  | 6 +++++-
 scheduler/src/lib.rs | 2 +-
 2 files changed, 6 insertions(+), 2 deletions(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 622b1bc652faad..258e31cd0e7e87 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1309,8 +1309,12 @@ impl Scheduler<ExecuteTimings> {
         let mut address_book = solana_scheduler::AddressBook::default();
         let preloader = Arc::new(address_book.preloader());
         let (transaction_sender, transaction_receiver) = crossbeam_channel::unbounded();
+
         let (scheduled_ee_sender, scheduled_ee_receiver) = crossbeam_channel::unbounded();
         let (scheduled_high_ee_sender, scheduled_high_ee_receiver) = crossbeam_channel::unbounded();
+        scheduled_ee_sender.send(solana_scheduler::ExecutablePayload(solana_scheduler:SpinWaitable::Spin)).unwrap();
+        scheduled_high_ee_sender.send(solana_scheduler::ExecutablePayload(solana_scheduler:SpinWaitable::Spin)).unwrap();
+
         let (processed_ee_sender, processed_ee_receiver) = crossbeam_channel::unbounded();
         let (retired_ee_sender, retired_ee_receiver) = crossbeam_channel::unbounded();
 
@@ -1352,7 +1356,7 @@ impl Scheduler<ExecuteTimings> {
                                 continue;
                             },
                             Ok(solana_scheduler::ExecutablePayload(solana_scheduler::SpinWaitable::Payload(ee))) => {
-                                s.send(solana_scheduler::ExecutablePayload(solana_scheduler::SpinWaitable::Spin)).unwrap(); 
+                                s.send(solana_scheduler::ExecutablePayload(solana_scheduler::SpinWaitable::Spin)).unwrap();
                                 maybe_ee = Some(ee);
                                 break;
                             }
diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index f55153c45d843c..ec7bf1995cb6fe 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1780,7 +1780,7 @@ impl ScheduleStage {
                                assert_eq!(from_exec.len(), 0);
                                from_exec_disconnected = true;
                                info!("flushing1..: {:?} {} {} {} {}", (from_disconnected, from_exec_disconnected), channel_backed_runnable_queue.task_count_hint(), contended_count, executing_queue_count, provisioning_tracker_count);
-                               break;
+                               todo!(); //break;
                            },
                        }
                    }

From 1fbf13c7546a899d409ba5517002a25987cfdde9 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 8 Oct 2022 14:40:11 +0900
Subject: [PATCH 3161/3199] save

---
 runtime/src/bank.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 258e31cd0e7e87..1b44c278b5e08f 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1312,8 +1312,8 @@ impl Scheduler<ExecuteTimings> {
 
         let (scheduled_ee_sender, scheduled_ee_receiver) = crossbeam_channel::unbounded();
         let (scheduled_high_ee_sender, scheduled_high_ee_receiver) = crossbeam_channel::unbounded();
-        scheduled_ee_sender.send(solana_scheduler::ExecutablePayload(solana_scheduler:SpinWaitable::Spin)).unwrap();
-        scheduled_high_ee_sender.send(solana_scheduler::ExecutablePayload(solana_scheduler:SpinWaitable::Spin)).unwrap();
+        scheduled_ee_sender.send(solana_scheduler::ExecutablePayload(solana_scheduler::SpinWaitable::Spin)).unwrap();
+        scheduled_high_ee_sender.send(solana_scheduler::ExecutablePayload(solana_scheduler::SpinWaitable::Spin)).unwrap();
 
         let (processed_ee_sender, processed_ee_receiver) = crossbeam_channel::unbounded();
         let (retired_ee_sender, retired_ee_receiver) = crossbeam_channel::unbounded();

From 7fe073e8a8e3df111c8d4c8d0924e03b2de95d4b Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 8 Oct 2022 15:00:40 +0900
Subject: [PATCH 3162/3199] save

---
 scheduler/src/lib.rs | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index ec7bf1995cb6fe..a97dea290d4cf5 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -2119,7 +2119,8 @@ pub enum Flushable<T, C> {
     Flush(std::sync::Arc<Checkpoint<C>>),
 }
 
-pub enum SpinWaitable<T> {
+pub enum SpinWaitable<T, C> {
     Payload(T),
     Spin,
+    Flush(std::sync::Arc<Checkpoint<C>>),
 }

From 9537715e3f56493f66ba50d2e67f5c83a15d9928 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 8 Oct 2022 15:01:21 +0900
Subject: [PATCH 3163/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index a97dea290d4cf5..2c0e16d61a7289 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -2051,7 +2051,7 @@ impl ScheduleStage {
 }
 
 pub struct SchedulablePayload<C>(pub Flushable<TaskInQueue, C>);
-pub struct ExecutablePayload(pub SpinWaitable<Box<ExecutionEnvironment>>);
+pub struct ExecutablePayload<C>(pub SpinWaitable<Box<ExecutionEnvironment>, C>);
 pub struct UnlockablePayload<T>(pub Box<ExecutionEnvironment>, pub T);
 pub struct ExaminablePayload<T, C>(pub Flushable<(Box<ExecutionEnvironment>, T), C>);
 

From cc8597b51d32e6f72373f6ea6c98bc2b1a04430d Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 8 Oct 2022 15:01:41 +0900
Subject: [PATCH 3164/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 2c0e16d61a7289..bb5245bc5f49aa 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -2026,7 +2026,7 @@ impl ScheduleStage {
         address_book: &mut AddressBook,
         from: &crossbeam_channel::Receiver<SchedulablePayload<C>>,
         to_execute_substage: &crossbeam_channel::Sender<ExecutablePayload>,
-        to_high_execute_substage: Option<&crossbeam_channel::Sender<ExecutablePayload>>,
+        to_high_execute_substage: Option<&crossbeam_channel::Sender<ExecutablePayload<C>>>,
         from_execute_substage: &crossbeam_channel::Receiver<UnlockablePayload<T>>,
         maybe_to_next_stage: Option<&crossbeam_channel::Sender<ExaminablePayload<T, C>>>, // assume nonblocking
     ) -> Option<std::sync::Arc<Checkpoint<C>>> {

From b84d0ffc62f65131c6410667939d01683ae60d36 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 8 Oct 2022 15:01:50 +0900
Subject: [PATCH 3165/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index bb5245bc5f49aa..b7058cf33060b7 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -2025,7 +2025,7 @@ impl ScheduleStage {
         runnable_queue: &mut TaskQueue,
         address_book: &mut AddressBook,
         from: &crossbeam_channel::Receiver<SchedulablePayload<C>>,
-        to_execute_substage: &crossbeam_channel::Sender<ExecutablePayload>,
+        to_execute_substage: &crossbeam_channel::Sender<ExecutablePayload<C>>,
         to_high_execute_substage: Option<&crossbeam_channel::Sender<ExecutablePayload<C>>>,
         from_execute_substage: &crossbeam_channel::Receiver<UnlockablePayload<T>>,
         maybe_to_next_stage: Option<&crossbeam_channel::Sender<ExaminablePayload<T, C>>>, // assume nonblocking

From 747bd938bd1bdd4b53b289aaca1df712aa13315b Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 8 Oct 2022 15:01:58 +0900
Subject: [PATCH 3166/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index b7058cf33060b7..5c60d41e13c21a 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1614,7 +1614,7 @@ impl ScheduleStage {
         address_book: &mut AddressBook,
         mut from_prev: &'a crossbeam_channel::Receiver<SchedulablePayload<C>>,
         to_execute_substage: &crossbeam_channel::Sender<ExecutablePayload>,
-        to_high_execute_substage: Option<&crossbeam_channel::Sender<ExecutablePayload>>,
+        to_high_execute_substage: Option<&crossbeam_channel::Sender<ExecutablePayload<C>>>,
         from_exec: &crossbeam_channel::Receiver<UnlockablePayload<T>>,
         maybe_to_next_stage: Option<&crossbeam_channel::Sender<ExaminablePayload<T, C>>>, // assume nonblocking
         never: &'a crossbeam_channel::Receiver<SchedulablePayload<C>>,

From fdf48ec9983dab7e83a01d71ca3c3d31f3cc91b5 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 8 Oct 2022 15:02:06 +0900
Subject: [PATCH 3167/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 5c60d41e13c21a..c6e2460ed4bfc2 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1613,7 +1613,7 @@ impl ScheduleStage {
         _runnable_queue: &mut TaskQueue,
         address_book: &mut AddressBook,
         mut from_prev: &'a crossbeam_channel::Receiver<SchedulablePayload<C>>,
-        to_execute_substage: &crossbeam_channel::Sender<ExecutablePayload>,
+        to_execute_substage: &crossbeam_channel::Sender<ExecutablePayload<C>>,
         to_high_execute_substage: Option<&crossbeam_channel::Sender<ExecutablePayload<C>>>,
         from_exec: &crossbeam_channel::Receiver<UnlockablePayload<T>>,
         maybe_to_next_stage: Option<&crossbeam_channel::Sender<ExaminablePayload<T, C>>>, // assume nonblocking

From dd36a93a33910860ff89ee66175fbbca8d7e74ce Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 8 Oct 2022 15:06:42 +0900
Subject: [PATCH 3168/3199] save

---
 runtime/src/bank.rs | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 1b44c278b5e08f..87f96201aa81bb 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1363,6 +1363,9 @@ impl Scheduler<ExecuteTimings> {
                             Ok(solana_scheduler::ExecutablePayload(solana_scheduler::SpinWaitable::Spin)) => {
                                 unreachable!();
                             }
+                            Ok(solana_scheduler::ExecutablePayload(solana_scheduler::SpinWaitable::Flush(checkpoint))) => {
+                                checkpoint.wait_for_restart(None);
+                            }
                             Err(crossbeam_channel::TryRecvError::Disconnected) => {
                                 continue;
                             },
@@ -1372,6 +1375,9 @@ impl Scheduler<ExecuteTimings> {
                 Ok(solana_scheduler::ExecutablePayload(solana_scheduler::SpinWaitable::Payload(mut ee))) => {
                     maybe_ee = Some(ee);
                 },
+                Ok(solana_scheduler::ExecutablePayload(solana_scheduler::SpinWaitable::Flush(checkpoint))) => {
+                    checkpoint.wait_for_restart(None);
+                },
                 Err(_) => todo!(),
             }
 
@@ -1438,8 +1444,6 @@ impl Scheduler<ExecuteTimings> {
 
                 //ee.reindex_with_address_book();
                 processed_ee_sender.send(solana_scheduler::UnlockablePayload(ee, timings)).unwrap();
-            } else {
-                unreachable!();
             }
             }
             todo!();

From b8035f32c6b1545c0ec4ede11867e2a408eed234 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 8 Oct 2022 15:15:40 +0900
Subject: [PATCH 3169/3199] save

---
 runtime/src/bank.rs | 55 +++++++++++++++++++++++++++++++--------------
 1 file changed, 38 insertions(+), 17 deletions(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 87f96201aa81bb..2b80daa066e4d2 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1343,40 +1343,61 @@ impl Scheduler<ExecuteTimings> {
             let r = (if thx >= executing_thread_count { scheduled_high_ee_receiver } else { scheduled_ee_receiver});
             let s = (if thx >= executing_thread_count { scheduled_high_ee_sender } else { scheduled_ee_sender});
 
+            let mut observed_payload = false;
+
             loop {
             let mut maybe_ee = None;
 
-            let received = r.recv();
-            match received {
+            match r.recv() {
                 Ok(solana_scheduler::ExecutablePayload(solana_scheduler::SpinWaitable::Spin)) => {
-                    loop {
-                        match r.try_recv() {
-                            Err(crossbeam_channel::TryRecvError::Empty) => {
-                                // let's spin
-                                continue;
-                            },
-                            Ok(solana_scheduler::ExecutablePayload(solana_scheduler::SpinWaitable::Payload(ee))) => {
+                    if observed_payload {
+                        loop {
+                            match r.try_recv() {
+                                Err(crossbeam_channel::TryRecvError::Empty) => {
+                                    // let's spin
+                                    continue;
+                                },
+                                Ok(solana_scheduler::ExecutablePayload(solana_scheduler::SpinWaitable::Payload(ee))) => {
+                                    s.send(solana_scheduler::ExecutablePayload(solana_scheduler::SpinWaitable::Spin)).unwrap();
+                                    maybe_ee = Some(ee);
+                                    observed_payload = true;
+                                    break;
+                                }
+                                Ok(solana_scheduler::ExecutablePayload(solana_scheduler::SpinWaitable::Spin)) => {
+                                    unreachable!();
+                                }
+                                Ok(solana_scheduler::ExecutablePayload(solana_scheduler::SpinWaitable::Flush(checkpoint))) => {
+                                    checkpoint.wait_for_restart(None);
+                                    observed_payload = false;
+                                }
+                                Err(crossbeam_channel::TryRecvError::Disconnected) => {
+                                    continue;
+                                },
+                            }
+                        }
+                    } else {
+                        match r.recv() {
+                            Ok(solana_scheduler::ExecutablePayload(solana_scheduler::SpinWaitable::Spin)) => unreachable!(),
+                            Ok(solana_scheduler::ExecutablePayload(solana_scheduler::SpinWaitable::Payload(mut ee))) => {
                                 s.send(solana_scheduler::ExecutablePayload(solana_scheduler::SpinWaitable::Spin)).unwrap();
                                 maybe_ee = Some(ee);
-                                break;
-                            }
-                            Ok(solana_scheduler::ExecutablePayload(solana_scheduler::SpinWaitable::Spin)) => {
-                                unreachable!();
-                            }
+                                observed_payload = true;
+                            },
                             Ok(solana_scheduler::ExecutablePayload(solana_scheduler::SpinWaitable::Flush(checkpoint))) => {
                                 checkpoint.wait_for_restart(None);
+                                observed_payload = false;
                             }
-                            Err(crossbeam_channel::TryRecvError::Disconnected) => {
-                                continue;
-                            },
+                            Err(_) => todo!(),
                         }
                     }
                 },
                 Ok(solana_scheduler::ExecutablePayload(solana_scheduler::SpinWaitable::Payload(mut ee))) => {
                     maybe_ee = Some(ee);
+                    observed_payload = true;
                 },
                 Ok(solana_scheduler::ExecutablePayload(solana_scheduler::SpinWaitable::Flush(checkpoint))) => {
                     checkpoint.wait_for_restart(None);
+                    observed_payload = false;
                 },
                 Err(_) => todo!(),
             }

From b43642b232d76e420a8a5e21017eced3a95283ed Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 8 Oct 2022 15:20:13 +0900
Subject: [PATCH 3170/3199] save

---
 runtime/src/bank.rs | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 2b80daa066e4d2..34544c19189dee 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1257,6 +1257,8 @@ struct Scheduler<C> {
     executing_thread_handles: Option<Vec<std::thread::JoinHandle<Result<(Duration, Duration)>>>>,
     error_collector_thread_handle: Option<std::thread::JoinHandle<Result<(Duration, Duration)>>>,
     transaction_sender: Option<crossbeam_channel::Sender<solana_scheduler::SchedulablePayload<C>>>,
+    scheduled_ee_sender: crossbeam_channel::Sender<solana_scheduler::ExecutablePayload<C>>,
+    scheduled_high_ee_sender: crossbeam_channel::Sender<solana_scheduler::ExecutablePayload<C>>,
     preloader: Arc<solana_scheduler::Preloader>,
     graceful_stop_initiated: AtomicBool,
     collected_results: Arc<std::sync::Mutex<Vec<Result<C>>>>,
@@ -1597,6 +1599,8 @@ impl Scheduler<ExecuteTimings> {
             executing_thread_handles: Some(executing_thread_handles),
             error_collector_thread_handle: Some(error_collector_thread_handle),
             transaction_sender: Some(transaction_sender),
+            scheduled_ee_sender,
+            scheduled_high_ee_sender,
             preloader,
             graceful_stop_initiated: Default::default(),
             collected_results,
@@ -1635,7 +1639,7 @@ impl<C> Scheduler<C> {
         //let transaction_sender = self.transaction_sender.take().unwrap();
 
         //drop(transaction_sender);
-        let checkpoint = solana_scheduler::Checkpoint::new(3);
+        let checkpoint = solana_scheduler::Checkpoint::new(3 + self.executing_thread_handles.len());
         self.transaction_sender
             .as_ref()
             .unwrap()

From 5cfe05f694faa18226488fe7ffa3a5bb3dc2c062 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 8 Oct 2022 15:21:06 +0900
Subject: [PATCH 3171/3199] save

---
 runtime/src/bank.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 34544c19189dee..47fcb3631243b8 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1639,7 +1639,7 @@ impl<C> Scheduler<C> {
         //let transaction_sender = self.transaction_sender.take().unwrap();
 
         //drop(transaction_sender);
-        let checkpoint = solana_scheduler::Checkpoint::new(3 + self.executing_thread_handles.len());
+        let checkpoint = solana_scheduler::Checkpoint::new(3 + self.executing_thread_handles.unwrap().len());
         self.transaction_sender
             .as_ref()
             .unwrap()

From 1fb4f63a5269dd5d6b0119bd52b20aedf269c3d9 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 8 Oct 2022 15:22:34 +0900
Subject: [PATCH 3172/3199] save

---
 runtime/src/bank.rs | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 47fcb3631243b8..4ae0dfd5fd5d1b 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1314,6 +1314,7 @@ impl Scheduler<ExecuteTimings> {
 
         let (scheduled_ee_sender, scheduled_ee_receiver) = crossbeam_channel::unbounded();
         let (scheduled_high_ee_sender, scheduled_high_ee_receiver) = crossbeam_channel::unbounded();
+        let (scheduled_ee_sender2, scheduled_high_ee_sender2) = (scheduled_ee_sender.clone(), scheduled_high_ee_sender.clone());
         scheduled_ee_sender.send(solana_scheduler::ExecutablePayload(solana_scheduler::SpinWaitable::Spin)).unwrap();
         scheduled_high_ee_sender.send(solana_scheduler::ExecutablePayload(solana_scheduler::SpinWaitable::Spin)).unwrap();
 
@@ -1599,8 +1600,8 @@ impl Scheduler<ExecuteTimings> {
             executing_thread_handles: Some(executing_thread_handles),
             error_collector_thread_handle: Some(error_collector_thread_handle),
             transaction_sender: Some(transaction_sender),
-            scheduled_ee_sender,
-            scheduled_high_ee_sender,
+            scheduled_ee_sender: scheduled_ee_sender2,
+            scheduled_high_ee_sender: scheduled_high_ee_sender2,
             preloader,
             graceful_stop_initiated: Default::default(),
             collected_results,

From 859e826c60285ead48f53970e3b9278fc24306a8 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 8 Oct 2022 15:22:56 +0900
Subject: [PATCH 3173/3199] save

---
 runtime/src/bank.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 4ae0dfd5fd5d1b..9b65b79bef1e27 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1640,7 +1640,7 @@ impl<C> Scheduler<C> {
         //let transaction_sender = self.transaction_sender.take().unwrap();
 
         //drop(transaction_sender);
-        let checkpoint = solana_scheduler::Checkpoint::new(3 + self.executing_thread_handles.unwrap().len());
+        let checkpoint = solana_scheduler::Checkpoint::new(3 + self.executing_thread_handles.as_ref().unwrap().len());
         self.transaction_sender
             .as_ref()
             .unwrap()

From a088309178805741883d559ad4e98dc7276255d6 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 8 Oct 2022 15:25:35 +0900
Subject: [PATCH 3174/3199] save

---
 runtime/src/bank.rs | 16 +++++++++++++++-
 1 file changed, 15 insertions(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 9b65b79bef1e27..79f45af8acda67 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1640,7 +1640,9 @@ impl<C> Scheduler<C> {
         //let transaction_sender = self.transaction_sender.take().unwrap();
 
         //drop(transaction_sender);
-        let checkpoint = solana_scheduler::Checkpoint::new(3 + self.executing_thread_handles.as_ref().unwrap().len());
+        let lane_count = self.executing_thread_handles.as_ref().unwrap().len();
+
+        let checkpoint = solana_scheduler::Checkpoint::new(3 + lane_count);
         self.transaction_sender
             .as_ref()
             .unwrap()
@@ -1648,6 +1650,18 @@ impl<C> Scheduler<C> {
                 solana_scheduler::Flushable::Flush(std::sync::Arc::clone(&checkpoint)),
             ))
             .unwrap();
+        for i in 0..lane_count {
+            if i < lane_count/2 {
+                self.scheduled_ee_sender.send(solana_scheduler::ExecutablePayload(
+                solana_scheduler::SpinWaitable::Flush(std::sync::Arc::clone(&checkpoint)),
+            )).unwrap();
+            } else {
+                self.scheduled_high_ee_sender.send(solana_scheduler::ExecutablePayload(
+                solana_scheduler::SpinWaitable::Flush(std::sync::Arc::clone(&checkpoint)),
+            )).unwrap();
+            }
+        }
+
         checkpoint.wait_for_restart(None);
         let r = checkpoint.take_restart_value();
         self.collected_results.lock().unwrap().push(Ok(r));

From 81196f6709f773a750dc528c18cd8634d82a9f6c Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 8 Oct 2022 15:49:26 +0900
Subject: [PATCH 3175/3199] save

---
 runtime/src/bank.rs | 27 +++++++++++++++------------
 1 file changed, 15 insertions(+), 12 deletions(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 79f45af8acda67..9dc1c19cb1f82b 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1640,9 +1640,8 @@ impl<C> Scheduler<C> {
         //let transaction_sender = self.transaction_sender.take().unwrap();
 
         //drop(transaction_sender);
-        let lane_count = self.executing_thread_handles.as_ref().unwrap().len();
 
-        let checkpoint = solana_scheduler::Checkpoint::new(3 + lane_count);
+        let checkpoint = solana_scheduler::Checkpoint::new(3);
         self.transaction_sender
             .as_ref()
             .unwrap()
@@ -1650,25 +1649,29 @@ impl<C> Scheduler<C> {
                 solana_scheduler::Flushable::Flush(std::sync::Arc::clone(&checkpoint)),
             ))
             .unwrap();
+
+        checkpoint.wait_for_restart(None);
+        let r = checkpoint.take_restart_value();
+        self.collected_results.lock().unwrap().push(Ok(r));
+        {
+            *self.bank.write().unwrap() = None;
+            self.slot.store(0, std::sync::atomic::Ordering::SeqCst);
+        }
+
+        let lane_count = self.executing_thread_handles.as_ref().unwrap().len();
+        let checkpoint_exec = solana_scheduler::Checkpoint::new(1 + lane_count);
         for i in 0..lane_count {
             if i < lane_count/2 {
                 self.scheduled_ee_sender.send(solana_scheduler::ExecutablePayload(
-                solana_scheduler::SpinWaitable::Flush(std::sync::Arc::clone(&checkpoint)),
+                solana_scheduler::SpinWaitable::Flush(std::sync::Arc::clone(&checkpoint_exec)),
             )).unwrap();
             } else {
                 self.scheduled_high_ee_sender.send(solana_scheduler::ExecutablePayload(
-                solana_scheduler::SpinWaitable::Flush(std::sync::Arc::clone(&checkpoint)),
+                solana_scheduler::SpinWaitable::Flush(std::sync::Arc::clone(&checkpoint_exec)),
             )).unwrap();
             }
         }
-
-        checkpoint.wait_for_restart(None);
-        let r = checkpoint.take_restart_value();
-        self.collected_results.lock().unwrap().push(Ok(r));
-        {
-            *self.bank.write().unwrap() = None;
-            self.slot.store(0, std::sync::atomic::Ordering::SeqCst);
-        }
+        checkpoint_exec.wait_for_restart(None);
 
         /*
         let executing_thread_duration_pairs: Result<Vec<_>> = self.executing_thread_handles.take().unwrap().into_iter().map(|executing_thread_handle| {

From e24120cbfcda2c5766da12d5ea10824bfb083d79 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 8 Oct 2022 15:52:00 +0900
Subject: [PATCH 3176/3199] save

---
 runtime/src/bank.rs | 1 -
 1 file changed, 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 9dc1c19cb1f82b..9b8f7ae2559a9f 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1649,7 +1649,6 @@ impl<C> Scheduler<C> {
                 solana_scheduler::Flushable::Flush(std::sync::Arc::clone(&checkpoint)),
             ))
             .unwrap();
-
         checkpoint.wait_for_restart(None);
         let r = checkpoint.take_restart_value();
         self.collected_results.lock().unwrap().push(Ok(r));

From f3ea5c09db5e58815e9ca5da77f001746daba204 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 8 Oct 2022 15:58:56 +0900
Subject: [PATCH 3177/3199] save

---
 scheduler/src/lib.rs | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index c6e2460ed4bfc2..2a80c7b72bed19 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -2055,7 +2055,7 @@ pub struct ExecutablePayload<C>(pub SpinWaitable<Box<ExecutionEnvironment>, C>);
 pub struct UnlockablePayload<T>(pub Box<ExecutionEnvironment>, pub T);
 pub struct ExaminablePayload<T, C>(pub Flushable<(Box<ExecutionEnvironment>, T), C>);
 
-pub struct Checkpoint<T>(std::sync::Mutex<(usize, Option<T>)>, std::sync::Condvar);
+pub struct Checkpoint<T>(std::sync::Mutex<(usize, Option<T>)>, std::sync::Condvar, bool);
 
 impl<T> Checkpoint<T> {
     pub fn wait_for_restart(&self, maybe_given_restart_value: Option<T>) {
@@ -2077,7 +2077,7 @@ impl<T> Checkpoint<T> {
         }
 
         if *self_remaining_threads == 0 {
-            assert!(self_return_value.is_some());
+            assert!(self.2 || self_return_value.is_some());
             drop(self_remaining_threads);
             self.1.notify_all();
             trace!(
@@ -2106,10 +2106,11 @@ impl<T> Checkpoint<T> {
         self_return_value.take().unwrap()
     }
 
-    pub fn new(remaining_threads: usize) -> std::sync::Arc<Self> {
+    pub fn new(remaining_threads: usize, no_restart_value: bool) -> std::sync::Arc<Self> {
         std::sync::Arc::new(Self(
             std::sync::Mutex::new((remaining_threads, None)),
             std::sync::Condvar::new(),
+            no_restart_value,
         ))
     }
 }

From 215ae7512bf7e96820d4061c675e80cc99970b3b Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 8 Oct 2022 15:59:23 +0900
Subject: [PATCH 3178/3199] save

---
 runtime/src/bank.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 9b8f7ae2559a9f..d93abd48d2f7b6 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1641,7 +1641,7 @@ impl<C> Scheduler<C> {
 
         //drop(transaction_sender);
 
-        let checkpoint = solana_scheduler::Checkpoint::new(3);
+        let checkpoint = solana_scheduler::Checkpoint::new(3, false);
         self.transaction_sender
             .as_ref()
             .unwrap()
@@ -1658,7 +1658,7 @@ impl<C> Scheduler<C> {
         }
 
         let lane_count = self.executing_thread_handles.as_ref().unwrap().len();
-        let checkpoint_exec = solana_scheduler::Checkpoint::new(1 + lane_count);
+        let checkpoint_exec = solana_scheduler::Checkpoint::new(1 + lane_count, true);
         for i in 0..lane_count {
             if i < lane_count/2 {
                 self.scheduled_ee_sender.send(solana_scheduler::ExecutablePayload(

From dfed599608a30662ec051472a0f422da2f3a5427 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 8 Oct 2022 21:20:57 +0900
Subject: [PATCH 3179/3199] save

---
 runtime/src/bank.rs  | 9 +++++++--
 scheduler/src/lib.rs | 7 +++++++
 2 files changed, 14 insertions(+), 2 deletions(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index d93abd48d2f7b6..add1b080c8f272 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1361,9 +1361,10 @@ impl Scheduler<ExecuteTimings> {
                                     continue;
                                 },
                                 Ok(solana_scheduler::ExecutablePayload(solana_scheduler::SpinWaitable::Payload(ee))) => {
-                                    s.send(solana_scheduler::ExecutablePayload(solana_scheduler::SpinWaitable::Spin)).unwrap();
+                                    s.send_buffered(solana_scheduler::ExecutablePayload(solana_scheduler::SpinWaitable::Spin)).unwrap();
                                     maybe_ee = Some(ee);
                                     observed_payload = true;
+                                    info!("ex recv via spin");
                                     break;
                                 }
                                 Ok(solana_scheduler::ExecutablePayload(solana_scheduler::SpinWaitable::Spin)) => {
@@ -1382,9 +1383,10 @@ impl Scheduler<ExecuteTimings> {
                         match r.recv() {
                             Ok(solana_scheduler::ExecutablePayload(solana_scheduler::SpinWaitable::Spin)) => unreachable!(),
                             Ok(solana_scheduler::ExecutablePayload(solana_scheduler::SpinWaitable::Payload(mut ee))) => {
-                                s.send(solana_scheduler::ExecutablePayload(solana_scheduler::SpinWaitable::Spin)).unwrap();
+                                s.send_buffered(solana_scheduler::ExecutablePayload(solana_scheduler::SpinWaitable::Spin)).unwrap();
                                 maybe_ee = Some(ee);
                                 observed_payload = true;
+                                info!("ex recv via initial spin");
                             },
                             Ok(solana_scheduler::ExecutablePayload(solana_scheduler::SpinWaitable::Flush(checkpoint))) => {
                                 checkpoint.wait_for_restart(None);
@@ -1397,6 +1399,7 @@ impl Scheduler<ExecuteTimings> {
                 Ok(solana_scheduler::ExecutablePayload(solana_scheduler::SpinWaitable::Payload(mut ee))) => {
                     maybe_ee = Some(ee);
                     observed_payload = true;
+                    info!("ex recv via blocking recv");
                 },
                 Ok(solana_scheduler::ExecutablePayload(solana_scheduler::SpinWaitable::Flush(checkpoint))) => {
                     checkpoint.wait_for_restart(None);
@@ -1467,7 +1470,9 @@ impl Scheduler<ExecuteTimings> {
                 ee.execution_us = wall_time.as_us();
 
                 //ee.reindex_with_address_book();
+                info!("ex send begin");
                 processed_ee_sender.send(solana_scheduler::UnlockablePayload(ee, timings)).unwrap();
+                info!("ex send end");
             }
             }
             todo!();
diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 2a80c7b72bed19..6315b878db5b62 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1720,6 +1720,7 @@ impl ScheduleStage {
                             if let Ok(UnlockablePayload(mut processed_execution_environment, extra)) = maybe_from_exec {
                                 executing_queue_count = executing_queue_count.checked_sub(1).unwrap();
                                 processed_count = processed_count.checked_add(1).unwrap();
+                                info!("commit blocking recv");
                                 Self::commit_processed_execution(ast, &mut processed_execution_environment, address_book, &mut commit_clock, &mut provisioning_tracker_count);
                                 to_next_stage.send_buffered(ExaminablePayload(Flushable::Payload((processed_execution_environment, extra)))).unwrap();
                             } else {
@@ -1772,6 +1773,7 @@ impl ScheduleStage {
                            Ok(UnlockablePayload(mut processed_execution_environment, extra)) => {
                                executing_queue_count = executing_queue_count.checked_sub(1).unwrap();
                                processed_count = processed_count.checked_add(1).unwrap();
+                                info!("commit spinning recv");
                                Self::commit_processed_execution(ast, &mut processed_execution_environment, address_book, &mut commit_clock, &mut provisioning_tracker_count);
                                to_next_stage.send_buffered(ExaminablePayload(Flushable::Payload((processed_execution_environment, extra)))).unwrap();
                                break;
@@ -1832,10 +1834,12 @@ impl ScheduleStage {
                     );
                     if let Some(ee) = maybe_ee {
                         executing_queue_count = executing_queue_count.checked_add(1).unwrap();
+                        info!("sc send high begin");
                         to_high_execute_substage
                             .unwrap_or(to_execute_substage)
                             .send(ExecutablePayload(SpinWaitable::Payload(ee)))
                             .unwrap();
+                        info!("sc send high end");
                     }
                     debug!("schedule_once id_{:016x} [C] ch(prev: {}, exec: {}+{}|{}), r: {}, u/c: {}/{}, (imm+provi)/max: ({}+{})/{} s: {} l(s+f): {}+{}", random_id, (if from_disconnected { "-".to_string() } else { format!("{}", from_prev.len()) }), to_high_execute_substage.map(|t| format!("{}", t.len())).unwrap_or("-".into()), to_execute_substage.len(), from_exec.len(), channel_backed_runnable_queue.task_count_hint(), address_book.uncontended_task_ids.len(), contended_count, executing_queue_count, provisioning_tracker_count, max_executing_queue_count, address_book.stuck_tasks.len(), processed_count, failed_lock_count);
 
@@ -1881,7 +1885,9 @@ impl ScheduleStage {
                     );
                     if let Some(ee) = maybe_ee {
                         executing_queue_count = executing_queue_count.checked_add(1).unwrap();
+                        info!("sc send begin");
                         to_execute_substage.send(ExecutablePayload(SpinWaitable::Payload(ee))).unwrap();
+                        info!("sc send end");
                     }
                     debug!("schedule_once id_{:016x} [R] ch(prev: {}, exec: {}+{}|{}), r: {}, u/c: {}/{}, (imm+provi)/max: ({}+{})/{} s: {} l(s+f): {}+{}", random_id, (if from_disconnected { "-".to_string() } else { format!("{}", from_prev.len()) }), to_high_execute_substage.map(|t| format!("{}", t.len())).unwrap_or("-".into()), to_execute_substage.len(), from_exec.len(), channel_backed_runnable_queue.task_count_hint(), address_book.uncontended_task_ids.len(), contended_count, executing_queue_count, provisioning_tracker_count, max_executing_queue_count, address_book.stuck_tasks.len(), processed_count, failed_lock_count);
 
@@ -1940,6 +1946,7 @@ impl ScheduleStage {
                         empty_from_exec = from_exec_len == 0;
                         executing_queue_count = executing_queue_count.checked_sub(1).unwrap();
                         processed_count = processed_count.checked_add(1).unwrap();
+                        info!("commit buffered recv");
                         Self::commit_processed_execution(
                             ast,
                             &mut processed_execution_environment,

From dd3197ac0979ceaf6a78b8ff373d9014a1734a21 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 8 Oct 2022 21:34:10 +0900
Subject: [PATCH 3180/3199] save

---
 runtime/src/bank.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index add1b080c8f272..ec3c1d74366b86 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1471,7 +1471,7 @@ impl Scheduler<ExecuteTimings> {
 
                 //ee.reindex_with_address_book();
                 info!("ex send begin");
-                processed_ee_sender.send(solana_scheduler::UnlockablePayload(ee, timings)).unwrap();
+                processed_ee_sender.send_buffered(solana_scheduler::UnlockablePayload(ee, timings)).unwrap();
                 info!("ex send end");
             }
             }

From 7df8e5959a812535bc619b75cca8604972e5ae8c Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 8 Oct 2022 21:35:04 +0900
Subject: [PATCH 3181/3199] save

---
 scheduler/src/lib.rs | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 6315b878db5b62..897ac48d2774ea 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1720,7 +1720,7 @@ impl ScheduleStage {
                             if let Ok(UnlockablePayload(mut processed_execution_environment, extra)) = maybe_from_exec {
                                 executing_queue_count = executing_queue_count.checked_sub(1).unwrap();
                                 processed_count = processed_count.checked_add(1).unwrap();
-                                info!("commit blocking recv");
+                                info!("sc blocking recv");
                                 Self::commit_processed_execution(ast, &mut processed_execution_environment, address_book, &mut commit_clock, &mut provisioning_tracker_count);
                                 to_next_stage.send_buffered(ExaminablePayload(Flushable::Payload((processed_execution_environment, extra)))).unwrap();
                             } else {
@@ -1773,7 +1773,7 @@ impl ScheduleStage {
                            Ok(UnlockablePayload(mut processed_execution_environment, extra)) => {
                                executing_queue_count = executing_queue_count.checked_sub(1).unwrap();
                                processed_count = processed_count.checked_add(1).unwrap();
-                                info!("commit spinning recv");
+                                info!("sc spinning recv");
                                Self::commit_processed_execution(ast, &mut processed_execution_environment, address_book, &mut commit_clock, &mut provisioning_tracker_count);
                                to_next_stage.send_buffered(ExaminablePayload(Flushable::Payload((processed_execution_environment, extra)))).unwrap();
                                break;
@@ -1946,7 +1946,7 @@ impl ScheduleStage {
                         empty_from_exec = from_exec_len == 0;
                         executing_queue_count = executing_queue_count.checked_sub(1).unwrap();
                         processed_count = processed_count.checked_add(1).unwrap();
-                        info!("commit buffered recv");
+                        info!("sc buffered recv");
                         Self::commit_processed_execution(
                             ast,
                             &mut processed_execution_environment,

From 8a99ef20e8c5f06161b21d72912a2c4182c48651 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 8 Oct 2022 21:36:43 +0900
Subject: [PATCH 3182/3199] save

---
 scheduler/src/lib.rs | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 897ac48d2774ea..91df34ae884b26 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1720,7 +1720,7 @@ impl ScheduleStage {
                             if let Ok(UnlockablePayload(mut processed_execution_environment, extra)) = maybe_from_exec {
                                 executing_queue_count = executing_queue_count.checked_sub(1).unwrap();
                                 processed_count = processed_count.checked_add(1).unwrap();
-                                info!("sc blocking recv");
+                                info!("    sc blocking recv");
                                 Self::commit_processed_execution(ast, &mut processed_execution_environment, address_book, &mut commit_clock, &mut provisioning_tracker_count);
                                 to_next_stage.send_buffered(ExaminablePayload(Flushable::Payload((processed_execution_environment, extra)))).unwrap();
                             } else {
@@ -1773,7 +1773,7 @@ impl ScheduleStage {
                            Ok(UnlockablePayload(mut processed_execution_environment, extra)) => {
                                executing_queue_count = executing_queue_count.checked_sub(1).unwrap();
                                processed_count = processed_count.checked_add(1).unwrap();
-                                info!("sc spinning recv");
+                                info!("    sc spinning recv");
                                Self::commit_processed_execution(ast, &mut processed_execution_environment, address_book, &mut commit_clock, &mut provisioning_tracker_count);
                                to_next_stage.send_buffered(ExaminablePayload(Flushable::Payload((processed_execution_environment, extra)))).unwrap();
                                break;
@@ -1834,12 +1834,12 @@ impl ScheduleStage {
                     );
                     if let Some(ee) = maybe_ee {
                         executing_queue_count = executing_queue_count.checked_add(1).unwrap();
-                        info!("sc send high begin");
+                        info!("    sc send high begin");
                         to_high_execute_substage
                             .unwrap_or(to_execute_substage)
                             .send(ExecutablePayload(SpinWaitable::Payload(ee)))
                             .unwrap();
-                        info!("sc send high end");
+                        info!("    sc send high end");
                     }
                     debug!("schedule_once id_{:016x} [C] ch(prev: {}, exec: {}+{}|{}), r: {}, u/c: {}/{}, (imm+provi)/max: ({}+{})/{} s: {} l(s+f): {}+{}", random_id, (if from_disconnected { "-".to_string() } else { format!("{}", from_prev.len()) }), to_high_execute_substage.map(|t| format!("{}", t.len())).unwrap_or("-".into()), to_execute_substage.len(), from_exec.len(), channel_backed_runnable_queue.task_count_hint(), address_book.uncontended_task_ids.len(), contended_count, executing_queue_count, provisioning_tracker_count, max_executing_queue_count, address_book.stuck_tasks.len(), processed_count, failed_lock_count);
 
@@ -1885,9 +1885,9 @@ impl ScheduleStage {
                     );
                     if let Some(ee) = maybe_ee {
                         executing_queue_count = executing_queue_count.checked_add(1).unwrap();
-                        info!("sc send begin");
+                        info!("    sc send begin");
                         to_execute_substage.send(ExecutablePayload(SpinWaitable::Payload(ee))).unwrap();
-                        info!("sc send end");
+                        info!("    sc send end");
                     }
                     debug!("schedule_once id_{:016x} [R] ch(prev: {}, exec: {}+{}|{}), r: {}, u/c: {}/{}, (imm+provi)/max: ({}+{})/{} s: {} l(s+f): {}+{}", random_id, (if from_disconnected { "-".to_string() } else { format!("{}", from_prev.len()) }), to_high_execute_substage.map(|t| format!("{}", t.len())).unwrap_or("-".into()), to_execute_substage.len(), from_exec.len(), channel_backed_runnable_queue.task_count_hint(), address_book.uncontended_task_ids.len(), contended_count, executing_queue_count, provisioning_tracker_count, max_executing_queue_count, address_book.stuck_tasks.len(), processed_count, failed_lock_count);
 
@@ -1946,7 +1946,7 @@ impl ScheduleStage {
                         empty_from_exec = from_exec_len == 0;
                         executing_queue_count = executing_queue_count.checked_sub(1).unwrap();
                         processed_count = processed_count.checked_add(1).unwrap();
-                        info!("sc buffered recv");
+                        info!("    sc buffered recv");
                         Self::commit_processed_execution(
                             ast,
                             &mut processed_execution_environment,

From 5025ef763e3e1ced19697b2d06ba4ff40898aa6d Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 8 Oct 2022 22:16:02 +0900
Subject: [PATCH 3183/3199] save

---
 runtime/src/bank.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index ec3c1d74366b86..add1b080c8f272 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1471,7 +1471,7 @@ impl Scheduler<ExecuteTimings> {
 
                 //ee.reindex_with_address_book();
                 info!("ex send begin");
-                processed_ee_sender.send_buffered(solana_scheduler::UnlockablePayload(ee, timings)).unwrap();
+                processed_ee_sender.send(solana_scheduler::UnlockablePayload(ee, timings)).unwrap();
                 info!("ex send end");
             }
             }

From 0bc43a55cd56cc35d13503b6cd55897b392c0e2f Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 8 Oct 2022 22:17:38 +0900
Subject: [PATCH 3184/3199] save

---
 runtime/src/bank.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index add1b080c8f272..4219f7cf7e8677 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1428,7 +1428,7 @@ impl Scheduler<ExecuteTimings> {
                     TransactionBatch::new(vec![lock_result], &bank, Cow::Owned(vec![ee.task.tx.0.clone()]));
                 batch.set_needs_unlock(false);
 
-                let mut timings = Default::default();
+                let mut timings = Box::new(Default::default());
                 let (tx_results, _balances) = bank.load_execute_and_commit_transactions(
                     &batch,
                     MAX_PROCESSING_AGE,

From d0efab6d36527b17d27f1a38a88009617fb778f2 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 8 Oct 2022 22:20:24 +0900
Subject: [PATCH 3185/3199] save

---
 runtime/src/bank.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 4219f7cf7e8677..e631d3e4bd1832 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1319,7 +1319,7 @@ impl Scheduler<ExecuteTimings> {
         scheduled_high_ee_sender.send(solana_scheduler::ExecutablePayload(solana_scheduler::SpinWaitable::Spin)).unwrap();
 
         let (processed_ee_sender, processed_ee_receiver) = crossbeam_channel::unbounded();
-        let (retired_ee_sender, retired_ee_receiver) = crossbeam_channel::unbounded();
+        let (retired_ee_sender, retired_ee_receiver) = crossbeam_channel::unbounded::<usize>();
 
         let bank = Arc::new(std::sync::RwLock::new(None::<std::sync::Weak<Bank>>));
 

From ca9f2316c82804b8bca82374bcc59f89666e3b38 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 8 Oct 2022 22:20:55 +0900
Subject: [PATCH 3186/3199] save

---
 runtime/src/bank.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index e631d3e4bd1832..3ec0165ba32914 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1319,7 +1319,7 @@ impl Scheduler<ExecuteTimings> {
         scheduled_high_ee_sender.send(solana_scheduler::ExecutablePayload(solana_scheduler::SpinWaitable::Spin)).unwrap();
 
         let (processed_ee_sender, processed_ee_receiver) = crossbeam_channel::unbounded();
-        let (retired_ee_sender, retired_ee_receiver) = crossbeam_channel::unbounded::<usize>();
+        let (retired_ee_sender, retired_ee_receiver) = crossbeam_channel::unbounded::<solana_scheduler::ExaminablePayload<Box<ExecuteTimings>, _>>();
 
         let bank = Arc::new(std::sync::RwLock::new(None::<std::sync::Weak<Bank>>));
 

From 58180434347eb204eca617d0515568baa76cb3c4 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 8 Oct 2022 22:31:54 +0900
Subject: [PATCH 3187/3199] save

---
 runtime/src/bank.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 3ec0165ba32914..1694c8120521f7 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1361,7 +1361,7 @@ impl Scheduler<ExecuteTimings> {
                                     continue;
                                 },
                                 Ok(solana_scheduler::ExecutablePayload(solana_scheduler::SpinWaitable::Payload(ee))) => {
-                                    s.send_buffered(solana_scheduler::ExecutablePayload(solana_scheduler::SpinWaitable::Spin)).unwrap();
+                                    s.send(solana_scheduler::ExecutablePayload(solana_scheduler::SpinWaitable::Spin)).unwrap();
                                     maybe_ee = Some(ee);
                                     observed_payload = true;
                                     info!("ex recv via spin");
@@ -1383,7 +1383,7 @@ impl Scheduler<ExecuteTimings> {
                         match r.recv() {
                             Ok(solana_scheduler::ExecutablePayload(solana_scheduler::SpinWaitable::Spin)) => unreachable!(),
                             Ok(solana_scheduler::ExecutablePayload(solana_scheduler::SpinWaitable::Payload(mut ee))) => {
-                                s.send_buffered(solana_scheduler::ExecutablePayload(solana_scheduler::SpinWaitable::Spin)).unwrap();
+                                s.send(solana_scheduler::ExecutablePayload(solana_scheduler::SpinWaitable::Spin)).unwrap();
                                 maybe_ee = Some(ee);
                                 observed_payload = true;
                                 info!("ex recv via initial spin");

From 67ed4cceb9c0813aff4455bd176c9198556243d4 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 8 Oct 2022 22:32:01 +0900
Subject: [PATCH 3188/3199] save

---
 scheduler/src/lib.rs | 1 +
 1 file changed, 1 insertion(+)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 91df34ae884b26..1540991f3d7d3d 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1768,6 +1768,7 @@ impl ScheduleStage {
                         match from_exec.try_recv() {
                            Err(crossbeam_channel::TryRecvError::Empty) => {
                                // let's spin
+                               std::thread::sleep(Duration::from_micros(2));
                                continue;
                            },
                            Ok(UnlockablePayload(mut processed_execution_environment, extra)) => {

From 3c638a2817354d028702b904a4a18d894507c194 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 8 Oct 2022 22:32:13 +0900
Subject: [PATCH 3189/3199] save

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 1540991f3d7d3d..0190c84da7600b 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1768,7 +1768,7 @@ impl ScheduleStage {
                         match from_exec.try_recv() {
                            Err(crossbeam_channel::TryRecvError::Empty) => {
                                // let's spin
-                               std::thread::sleep(Duration::from_micros(2));
+                               std::thread::sleep(std::time::Duration::from_micros(2));
                                continue;
                            },
                            Ok(UnlockablePayload(mut processed_execution_environment, extra)) => {

From 49f343da42e55cc916af114e59c38131bd3f4281 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 8 Oct 2022 22:44:01 +0900
Subject: [PATCH 3190/3199] save

---
 runtime/src/bank.rs  | 14 +++++++-------
 scheduler/src/lib.rs | 12 ++++++------
 2 files changed, 13 insertions(+), 13 deletions(-)

diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 1694c8120521f7..d42b604483ae8a 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1361,10 +1361,10 @@ impl Scheduler<ExecuteTimings> {
                                     continue;
                                 },
                                 Ok(solana_scheduler::ExecutablePayload(solana_scheduler::SpinWaitable::Payload(ee))) => {
-                                    s.send(solana_scheduler::ExecutablePayload(solana_scheduler::SpinWaitable::Spin)).unwrap();
+                                    s.send_buffered(solana_scheduler::ExecutablePayload(solana_scheduler::SpinWaitable::Spin)).unwrap();
                                     maybe_ee = Some(ee);
                                     observed_payload = true;
-                                    info!("ex recv via spin");
+                                    //info!("ex recv via spin");
                                     break;
                                 }
                                 Ok(solana_scheduler::ExecutablePayload(solana_scheduler::SpinWaitable::Spin)) => {
@@ -1383,10 +1383,10 @@ impl Scheduler<ExecuteTimings> {
                         match r.recv() {
                             Ok(solana_scheduler::ExecutablePayload(solana_scheduler::SpinWaitable::Spin)) => unreachable!(),
                             Ok(solana_scheduler::ExecutablePayload(solana_scheduler::SpinWaitable::Payload(mut ee))) => {
-                                s.send(solana_scheduler::ExecutablePayload(solana_scheduler::SpinWaitable::Spin)).unwrap();
+                                s.send_buffered(solana_scheduler::ExecutablePayload(solana_scheduler::SpinWaitable::Spin)).unwrap();
                                 maybe_ee = Some(ee);
                                 observed_payload = true;
-                                info!("ex recv via initial spin");
+                                //info!("ex recv via initial spin");
                             },
                             Ok(solana_scheduler::ExecutablePayload(solana_scheduler::SpinWaitable::Flush(checkpoint))) => {
                                 checkpoint.wait_for_restart(None);
@@ -1399,7 +1399,7 @@ impl Scheduler<ExecuteTimings> {
                 Ok(solana_scheduler::ExecutablePayload(solana_scheduler::SpinWaitable::Payload(mut ee))) => {
                     maybe_ee = Some(ee);
                     observed_payload = true;
-                    info!("ex recv via blocking recv");
+                    //info!("ex recv via blocking recv");
                 },
                 Ok(solana_scheduler::ExecutablePayload(solana_scheduler::SpinWaitable::Flush(checkpoint))) => {
                     checkpoint.wait_for_restart(None);
@@ -1470,9 +1470,9 @@ impl Scheduler<ExecuteTimings> {
                 ee.execution_us = wall_time.as_us();
 
                 //ee.reindex_with_address_book();
-                info!("ex send begin");
+                //info!("ex send begin");
                 processed_ee_sender.send(solana_scheduler::UnlockablePayload(ee, timings)).unwrap();
-                info!("ex send end");
+                //info!("ex send end");
             }
             }
             todo!();
diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 0190c84da7600b..e0c6833a2fa97d 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1768,13 +1768,13 @@ impl ScheduleStage {
                         match from_exec.try_recv() {
                            Err(crossbeam_channel::TryRecvError::Empty) => {
                                // let's spin
-                               std::thread::sleep(std::time::Duration::from_micros(2));
+                               //std::thread::sleep(std::time::Duration::from_micros(2));
                                continue;
                            },
                            Ok(UnlockablePayload(mut processed_execution_environment, extra)) => {
                                executing_queue_count = executing_queue_count.checked_sub(1).unwrap();
                                processed_count = processed_count.checked_add(1).unwrap();
-                                info!("    sc spinning recv");
+                               //info!("    sc spinning recv");
                                Self::commit_processed_execution(ast, &mut processed_execution_environment, address_book, &mut commit_clock, &mut provisioning_tracker_count);
                                to_next_stage.send_buffered(ExaminablePayload(Flushable::Payload((processed_execution_environment, extra)))).unwrap();
                                break;
@@ -1835,12 +1835,12 @@ impl ScheduleStage {
                     );
                     if let Some(ee) = maybe_ee {
                         executing_queue_count = executing_queue_count.checked_add(1).unwrap();
-                        info!("    sc send high begin");
+                        //info!("    sc send high begin");
                         to_high_execute_substage
                             .unwrap_or(to_execute_substage)
                             .send(ExecutablePayload(SpinWaitable::Payload(ee)))
                             .unwrap();
-                        info!("    sc send high end");
+                        //info!("    sc send high end");
                     }
                     debug!("schedule_once id_{:016x} [C] ch(prev: {}, exec: {}+{}|{}), r: {}, u/c: {}/{}, (imm+provi)/max: ({}+{})/{} s: {} l(s+f): {}+{}", random_id, (if from_disconnected { "-".to_string() } else { format!("{}", from_prev.len()) }), to_high_execute_substage.map(|t| format!("{}", t.len())).unwrap_or("-".into()), to_execute_substage.len(), from_exec.len(), channel_backed_runnable_queue.task_count_hint(), address_book.uncontended_task_ids.len(), contended_count, executing_queue_count, provisioning_tracker_count, max_executing_queue_count, address_book.stuck_tasks.len(), processed_count, failed_lock_count);
 
@@ -1886,9 +1886,9 @@ impl ScheduleStage {
                     );
                     if let Some(ee) = maybe_ee {
                         executing_queue_count = executing_queue_count.checked_add(1).unwrap();
-                        info!("    sc send begin");
+                        //info!("    sc send begin");
                         to_execute_substage.send(ExecutablePayload(SpinWaitable::Payload(ee))).unwrap();
-                        info!("    sc send end");
+                        //info!("    sc send end");
                     }
                     debug!("schedule_once id_{:016x} [R] ch(prev: {}, exec: {}+{}|{}), r: {}, u/c: {}/{}, (imm+provi)/max: ({}+{})/{} s: {} l(s+f): {}+{}", random_id, (if from_disconnected { "-".to_string() } else { format!("{}", from_prev.len()) }), to_high_execute_substage.map(|t| format!("{}", t.len())).unwrap_or("-".into()), to_execute_substage.len(), from_exec.len(), channel_backed_runnable_queue.task_count_hint(), address_book.uncontended_task_ids.len(), contended_count, executing_queue_count, provisioning_tracker_count, max_executing_queue_count, address_book.stuck_tasks.len(), processed_count, failed_lock_count);
 

From 2d7ebbaa2207ec57d2e7e09241d7403993f455c6 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 8 Oct 2022 22:48:06 +0900
Subject: [PATCH 3191/3199] save

---
 scheduler/src/lib.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index e0c6833a2fa97d..fddb7f513869b9 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1720,7 +1720,7 @@ impl ScheduleStage {
                             if let Ok(UnlockablePayload(mut processed_execution_environment, extra)) = maybe_from_exec {
                                 executing_queue_count = executing_queue_count.checked_sub(1).unwrap();
                                 processed_count = processed_count.checked_add(1).unwrap();
-                                info!("    sc blocking recv");
+                                //info!("    sc blocking recv");
                                 Self::commit_processed_execution(ast, &mut processed_execution_environment, address_book, &mut commit_clock, &mut provisioning_tracker_count);
                                 to_next_stage.send_buffered(ExaminablePayload(Flushable::Payload((processed_execution_environment, extra)))).unwrap();
                             } else {
@@ -1947,7 +1947,7 @@ impl ScheduleStage {
                         empty_from_exec = from_exec_len == 0;
                         executing_queue_count = executing_queue_count.checked_sub(1).unwrap();
                         processed_count = processed_count.checked_add(1).unwrap();
-                        info!("    sc buffered recv");
+                        //info!("    sc pending recv");
                         Self::commit_processed_execution(
                             ast,
                             &mut processed_execution_environment,

From ca0608c76f51b9d4d24f3bd96b2f8b7266062107 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 8 Oct 2022 22:54:54 +0900
Subject: [PATCH 3192/3199] Revert...

---
 ledger-tool/src/main.rs |  73 ++++++++++-----------
 runtime/src/bank.rs     | 102 ++---------------------------
 scheduler/src/lib.rs    | 141 +++++++++++++++-------------------------
 3 files changed, 91 insertions(+), 225 deletions(-)

diff --git a/ledger-tool/src/main.rs b/ledger-tool/src/main.rs
index a1952de504531e..57bda5bf6d2088 100644
--- a/ledger-tool/src/main.rs
+++ b/ledger-tool/src/main.rs
@@ -316,45 +316,44 @@ fn output_slot(
                     let send_metrics = std::env::var("SEND_METRICS").is_ok();
 
                     for step in 0.. {
-                        if let solana_scheduler::SpinWaitable::Payload(mut ee) = pre_execute_env_receiver.recv().unwrap().0 {
-                            if step % 1966 == 0 {
-                                error!("executing!: {} {}", step, pre_execute_env_receiver.len());
-                            }
-
-                            if send_metrics {
-                                let mut process_message_time = Measure::start("process_message_time");
-                                let sig = ee.task.tx.0.signature().to_string();
-                                trace!("execute substage: #{} {:#?}", step, &sig);
-                                std::thread::sleep(std::time::Duration::from_micros(
-                                    ee.cu.try_into().unwrap(),
-                                ));
-
-                                process_message_time.stop();
-                                let duration_with_overhead = process_message_time.as_us();
-
-                                datapoint_info!(
-                                    "individual_tx_stats",
-                                    ("slot", 33333, i64),
-                                    ("thread", current_thread_name, String),
-                                    ("signature", &sig, String),
-                                    ("account_locks_in_json", "{}", String),
-                                    ("status", "Ok", String),
-                                    ("duration", duration_with_overhead, i64),
-                                    ("compute_units", ee.cu, i64),
-                                );
-                            }
+                        let mut ee = pre_execute_env_receiver.recv().unwrap().0;
+                        if step % 1966 == 0 {
+                            error!("executing!: {} {}", step, pre_execute_env_receiver.len());
+                        }
 
-                            /*
-                            muxed_sender
-                                .send(solana_scheduler::Multiplexed::FromExecute(ee))
-                                .unwrap();
-                            */
-                            // ee.reindex_with_address_book();
-                            todo!("contended_write_task_count");
-                            post_execute_env_sender
-                                .send(solana_scheduler::UnlockablePayload(ee, ()))
-                                .unwrap();
+                        if send_metrics {
+                            let mut process_message_time = Measure::start("process_message_time");
+                            let sig = ee.task.tx.0.signature().to_string();
+                            trace!("execute substage: #{} {:#?}", step, &sig);
+                            std::thread::sleep(std::time::Duration::from_micros(
+                                ee.cu.try_into().unwrap(),
+                            ));
+
+                            process_message_time.stop();
+                            let duration_with_overhead = process_message_time.as_us();
+
+                            datapoint_info!(
+                                "individual_tx_stats",
+                                ("slot", 33333, i64),
+                                ("thread", current_thread_name, String),
+                                ("signature", &sig, String),
+                                ("account_locks_in_json", "{}", String),
+                                ("status", "Ok", String),
+                                ("duration", duration_with_overhead, i64),
+                                ("compute_units", ee.cu, i64),
+                            );
                         }
+
+                        /*
+                        muxed_sender
+                            .send(solana_scheduler::Multiplexed::FromExecute(ee))
+                            .unwrap();
+                        */
+                        // ee.reindex_with_address_book();
+                        todo!("contended_write_task_count");
+                        post_execute_env_sender
+                            .send(solana_scheduler::UnlockablePayload(ee, ()))
+                            .unwrap();
                     }
                 })
                 .unwrap();
diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index d42b604483ae8a..45ed9ad91c8145 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1257,8 +1257,6 @@ struct Scheduler<C> {
     executing_thread_handles: Option<Vec<std::thread::JoinHandle<Result<(Duration, Duration)>>>>,
     error_collector_thread_handle: Option<std::thread::JoinHandle<Result<(Duration, Duration)>>>,
     transaction_sender: Option<crossbeam_channel::Sender<solana_scheduler::SchedulablePayload<C>>>,
-    scheduled_ee_sender: crossbeam_channel::Sender<solana_scheduler::ExecutablePayload<C>>,
-    scheduled_high_ee_sender: crossbeam_channel::Sender<solana_scheduler::ExecutablePayload<C>>,
     preloader: Arc<solana_scheduler::Preloader>,
     graceful_stop_initiated: AtomicBool,
     collected_results: Arc<std::sync::Mutex<Vec<Result<C>>>>,
@@ -1311,15 +1309,10 @@ impl Scheduler<ExecuteTimings> {
         let mut address_book = solana_scheduler::AddressBook::default();
         let preloader = Arc::new(address_book.preloader());
         let (transaction_sender, transaction_receiver) = crossbeam_channel::unbounded();
-
         let (scheduled_ee_sender, scheduled_ee_receiver) = crossbeam_channel::unbounded();
         let (scheduled_high_ee_sender, scheduled_high_ee_receiver) = crossbeam_channel::unbounded();
-        let (scheduled_ee_sender2, scheduled_high_ee_sender2) = (scheduled_ee_sender.clone(), scheduled_high_ee_sender.clone());
-        scheduled_ee_sender.send(solana_scheduler::ExecutablePayload(solana_scheduler::SpinWaitable::Spin)).unwrap();
-        scheduled_high_ee_sender.send(solana_scheduler::ExecutablePayload(solana_scheduler::SpinWaitable::Spin)).unwrap();
-
         let (processed_ee_sender, processed_ee_receiver) = crossbeam_channel::unbounded();
-        let (retired_ee_sender, retired_ee_receiver) = crossbeam_channel::unbounded::<solana_scheduler::ExaminablePayload<Box<ExecuteTimings>, _>>();
+        let (retired_ee_sender, retired_ee_receiver) = crossbeam_channel::unbounded();
 
         let bank = Arc::new(std::sync::RwLock::new(None::<std::sync::Weak<Bank>>));
 
@@ -1334,7 +1327,6 @@ impl Scheduler<ExecuteTimings> {
 
         let executing_thread_handles = (0..(executing_thread_count * 2)).map(|thx| {
             let (scheduled_ee_receiver, scheduled_high_ee_receiver, processed_ee_sender) = (scheduled_ee_receiver.clone(), scheduled_high_ee_receiver.clone(), processed_ee_sender.clone());
-            let (scheduled_ee_sender, scheduled_high_ee_sender) = (scheduled_ee_sender.clone(), scheduled_high_ee_sender.clone());
             let bank = bank.clone();
 
             std::thread::Builder::new().name(format!("solScExLane{:02}", thx)).spawn(move || {
@@ -1343,72 +1335,7 @@ impl Scheduler<ExecuteTimings> {
                 thread_priority::set_current_thread_priority(thread_priority::ThreadPriority::Max).unwrap();
             }
 
-            let r = (if thx >= executing_thread_count { scheduled_high_ee_receiver } else { scheduled_ee_receiver});
-            let s = (if thx >= executing_thread_count { scheduled_high_ee_sender } else { scheduled_ee_sender});
-
-            let mut observed_payload = false;
-
-            loop {
-            let mut maybe_ee = None;
-
-            match r.recv() {
-                Ok(solana_scheduler::ExecutablePayload(solana_scheduler::SpinWaitable::Spin)) => {
-                    if observed_payload {
-                        loop {
-                            match r.try_recv() {
-                                Err(crossbeam_channel::TryRecvError::Empty) => {
-                                    // let's spin
-                                    continue;
-                                },
-                                Ok(solana_scheduler::ExecutablePayload(solana_scheduler::SpinWaitable::Payload(ee))) => {
-                                    s.send_buffered(solana_scheduler::ExecutablePayload(solana_scheduler::SpinWaitable::Spin)).unwrap();
-                                    maybe_ee = Some(ee);
-                                    observed_payload = true;
-                                    //info!("ex recv via spin");
-                                    break;
-                                }
-                                Ok(solana_scheduler::ExecutablePayload(solana_scheduler::SpinWaitable::Spin)) => {
-                                    unreachable!();
-                                }
-                                Ok(solana_scheduler::ExecutablePayload(solana_scheduler::SpinWaitable::Flush(checkpoint))) => {
-                                    checkpoint.wait_for_restart(None);
-                                    observed_payload = false;
-                                }
-                                Err(crossbeam_channel::TryRecvError::Disconnected) => {
-                                    continue;
-                                },
-                            }
-                        }
-                    } else {
-                        match r.recv() {
-                            Ok(solana_scheduler::ExecutablePayload(solana_scheduler::SpinWaitable::Spin)) => unreachable!(),
-                            Ok(solana_scheduler::ExecutablePayload(solana_scheduler::SpinWaitable::Payload(mut ee))) => {
-                                s.send_buffered(solana_scheduler::ExecutablePayload(solana_scheduler::SpinWaitable::Spin)).unwrap();
-                                maybe_ee = Some(ee);
-                                observed_payload = true;
-                                //info!("ex recv via initial spin");
-                            },
-                            Ok(solana_scheduler::ExecutablePayload(solana_scheduler::SpinWaitable::Flush(checkpoint))) => {
-                                checkpoint.wait_for_restart(None);
-                                observed_payload = false;
-                            }
-                            Err(_) => todo!(),
-                        }
-                    }
-                },
-                Ok(solana_scheduler::ExecutablePayload(solana_scheduler::SpinWaitable::Payload(mut ee))) => {
-                    maybe_ee = Some(ee);
-                    observed_payload = true;
-                    //info!("ex recv via blocking recv");
-                },
-                Ok(solana_scheduler::ExecutablePayload(solana_scheduler::SpinWaitable::Flush(checkpoint))) => {
-                    checkpoint.wait_for_restart(None);
-                    observed_payload = false;
-                },
-                Err(_) => todo!(),
-            }
-
-            if let Some(mut ee) = maybe_ee {
+            while let Ok(solana_scheduler::ExecutablePayload(mut ee)) = (if thx >= executing_thread_count { scheduled_high_ee_receiver.recv() } else { scheduled_ee_receiver.recv()}) {
                 let (mut wall_time, cpu_time) = (Measure::start("process_message_time"), cpu_time::ThreadTime::now());
 
                 let current_execute_clock = ee.task.execute_time();
@@ -1428,7 +1355,7 @@ impl Scheduler<ExecuteTimings> {
                     TransactionBatch::new(vec![lock_result], &bank, Cow::Owned(vec![ee.task.tx.0.clone()]));
                 batch.set_needs_unlock(false);
 
-                let mut timings = Box::new(Default::default());
+                let mut timings = Default::default();
                 let (tx_results, _balances) = bank.load_execute_and_commit_transactions(
                     &batch,
                     MAX_PROCESSING_AGE,
@@ -1470,10 +1397,7 @@ impl Scheduler<ExecuteTimings> {
                 ee.execution_us = wall_time.as_us();
 
                 //ee.reindex_with_address_book();
-                //info!("ex send begin");
                 processed_ee_sender.send(solana_scheduler::UnlockablePayload(ee, timings)).unwrap();
-                //info!("ex send end");
-            }
             }
             todo!();
 
@@ -1605,8 +1529,6 @@ impl Scheduler<ExecuteTimings> {
             executing_thread_handles: Some(executing_thread_handles),
             error_collector_thread_handle: Some(error_collector_thread_handle),
             transaction_sender: Some(transaction_sender),
-            scheduled_ee_sender: scheduled_ee_sender2,
-            scheduled_high_ee_sender: scheduled_high_ee_sender2,
             preloader,
             graceful_stop_initiated: Default::default(),
             collected_results,
@@ -1645,8 +1567,7 @@ impl<C> Scheduler<C> {
         //let transaction_sender = self.transaction_sender.take().unwrap();
 
         //drop(transaction_sender);
-
-        let checkpoint = solana_scheduler::Checkpoint::new(3, false);
+        let checkpoint = solana_scheduler::Checkpoint::new(3);
         self.transaction_sender
             .as_ref()
             .unwrap()
@@ -1662,21 +1583,6 @@ impl<C> Scheduler<C> {
             self.slot.store(0, std::sync::atomic::Ordering::SeqCst);
         }
 
-        let lane_count = self.executing_thread_handles.as_ref().unwrap().len();
-        let checkpoint_exec = solana_scheduler::Checkpoint::new(1 + lane_count, true);
-        for i in 0..lane_count {
-            if i < lane_count/2 {
-                self.scheduled_ee_sender.send(solana_scheduler::ExecutablePayload(
-                solana_scheduler::SpinWaitable::Flush(std::sync::Arc::clone(&checkpoint_exec)),
-            )).unwrap();
-            } else {
-                self.scheduled_high_ee_sender.send(solana_scheduler::ExecutablePayload(
-                solana_scheduler::SpinWaitable::Flush(std::sync::Arc::clone(&checkpoint_exec)),
-            )).unwrap();
-            }
-        }
-        checkpoint_exec.wait_for_restart(None);
-
         /*
         let executing_thread_duration_pairs: Result<Vec<_>> = self.executing_thread_handles.take().unwrap().into_iter().map(|executing_thread_handle| {
             executing_thread_handle.join().unwrap().map(|u| (u.0.as_micros(), u.1.as_micros()))
diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index fddb7f513869b9..1bf1343aaac06c 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1613,8 +1613,8 @@ impl ScheduleStage {
         _runnable_queue: &mut TaskQueue,
         address_book: &mut AddressBook,
         mut from_prev: &'a crossbeam_channel::Receiver<SchedulablePayload<C>>,
-        to_execute_substage: &crossbeam_channel::Sender<ExecutablePayload<C>>,
-        to_high_execute_substage: Option<&crossbeam_channel::Sender<ExecutablePayload<C>>>,
+        to_execute_substage: &crossbeam_channel::Sender<ExecutablePayload>,
+        to_high_execute_substage: Option<&crossbeam_channel::Sender<ExecutablePayload>>,
         from_exec: &crossbeam_channel::Receiver<UnlockablePayload<T>>,
         maybe_to_next_stage: Option<&crossbeam_channel::Sender<ExaminablePayload<T, C>>>, // assume nonblocking
         never: &'a crossbeam_channel::Receiver<SchedulablePayload<C>>,
@@ -1714,76 +1714,49 @@ impl ScheduleStage {
             let mut select_skipped = false;
 
             if !from_disconnected || executing_queue_count >= 1 {
-                if !from_disconnected {
-                    crossbeam_channel::select! {
-                        recv(from_exec) -> maybe_from_exec => {
-                            if let Ok(UnlockablePayload(mut processed_execution_environment, extra)) = maybe_from_exec {
-                                executing_queue_count = executing_queue_count.checked_sub(1).unwrap();
-                                processed_count = processed_count.checked_add(1).unwrap();
-                                //info!("    sc blocking recv");
-                                Self::commit_processed_execution(ast, &mut processed_execution_environment, address_book, &mut commit_clock, &mut provisioning_tracker_count);
-                                to_next_stage.send_buffered(ExaminablePayload(Flushable::Payload((processed_execution_environment, extra)))).unwrap();
-                            } else {
-                                assert_eq!(from_exec.len(), 0);
-                                from_exec_disconnected = true;
-                                info!("flushing1..: {:?} {} {} {} {}", (from_disconnected, from_exec_disconnected), channel_backed_runnable_queue.task_count_hint(), contended_count, executing_queue_count, provisioning_tracker_count);
-                                if from_disconnected {
-                                    break;
-                                }
-                            }
-                        }
-                        recv(from_prev) -> maybe_from => {
-                            match maybe_from {
-                                Ok(SchedulablePayload(Flushable::Payload(task))) => {
-                                    if maybe_start_time.is_none() {
-                                         info!("schedule_once:initial id_{:016x}", random_id);
-                                        maybe_start_time = Some(std::time::Instant::now());
-                                        last_time = maybe_start_time.clone();
-                                    }
-                                    //Self::register_runnable_task(task, runnable_queue, &mut sequence_time);
-                                    channel_backed_runnable_queue.buffer(task);
-                                },
-                                Ok(SchedulablePayload(Flushable::Flush(checkpoint))) => {
-                                    assert_eq!(from_prev.len(), 0);
-                                    assert!(!from_disconnected);
-                                    from_disconnected = true;
-                                    from_prev = never;
-                                    trace!("flushing2..: {:?} {} {} {} {}", (from_disconnected, from_exec_disconnected), channel_backed_runnable_queue.task_count_hint(), contended_count, executing_queue_count, provisioning_tracker_count);
-                                    assert!(maybe_checkpoint.is_none());
-                                    maybe_checkpoint = Some(checkpoint);
-                                },
-                                Err(_) => {
-                                    assert_eq!(from_prev.len(), 0);
-                                    assert!(!from_disconnected);
-                                    assert!(maybe_checkpoint.is_none());
-                                    from_disconnected = true;
-                                    from_prev = never;
-                                    trace!("flushing2..: {:?} {} {} {} {}", (from_disconnected, from_exec_disconnected), channel_backed_runnable_queue.task_count_hint(), contended_count, executing_queue_count, provisioning_tracker_count);
-                                },
-                            }
-                        }
-                    }
-                } else {
-                    loop {
-                        match from_exec.try_recv() {
-                           Err(crossbeam_channel::TryRecvError::Empty) => {
-                               // let's spin
-                               //std::thread::sleep(std::time::Duration::from_micros(2));
-                               continue;
-                           },
-                           Ok(UnlockablePayload(mut processed_execution_environment, extra)) => {
-                               executing_queue_count = executing_queue_count.checked_sub(1).unwrap();
-                               processed_count = processed_count.checked_add(1).unwrap();
-                               //info!("    sc spinning recv");
-                               Self::commit_processed_execution(ast, &mut processed_execution_environment, address_book, &mut commit_clock, &mut provisioning_tracker_count);
-                               to_next_stage.send_buffered(ExaminablePayload(Flushable::Payload((processed_execution_environment, extra)))).unwrap();
+                crossbeam_channel::select! {
+                   recv(from_exec) -> maybe_from_exec => {
+                       if let Ok(UnlockablePayload(mut processed_execution_environment, extra)) = maybe_from_exec {
+                           executing_queue_count = executing_queue_count.checked_sub(1).unwrap();
+                           processed_count = processed_count.checked_add(1).unwrap();
+                           Self::commit_processed_execution(ast, &mut processed_execution_environment, address_book, &mut commit_clock, &mut provisioning_tracker_count);
+                           to_next_stage.send_buffered(ExaminablePayload(Flushable::Payload((processed_execution_environment, extra)))).unwrap();
+                       } else {
+                           assert_eq!(from_exec.len(), 0);
+                           from_exec_disconnected = true;
+                           info!("flushing1..: {:?} {} {} {} {}", (from_disconnected, from_exec_disconnected), channel_backed_runnable_queue.task_count_hint(), contended_count, executing_queue_count, provisioning_tracker_count);
+                           if from_disconnected {
                                break;
+                           }
+                       }
+                   }
+                   recv(from_prev) -> maybe_from => {
+                       match maybe_from {
+                           Ok(SchedulablePayload(Flushable::Payload(task))) => {
+                               if maybe_start_time.is_none() {
+                                    info!("schedule_once:initial id_{:016x}", random_id);
+                                   maybe_start_time = Some(std::time::Instant::now());
+                                   last_time = maybe_start_time.clone();
+                               }
+                               //Self::register_runnable_task(task, runnable_queue, &mut sequence_time);
+                               channel_backed_runnable_queue.buffer(task);
+                           },
+                           Ok(SchedulablePayload(Flushable::Flush(checkpoint))) => {
+                               assert_eq!(from_prev.len(), 0);
+                               assert!(!from_disconnected);
+                               from_disconnected = true;
+                               from_prev = never;
+                               trace!("flushing2..: {:?} {} {} {} {}", (from_disconnected, from_exec_disconnected), channel_backed_runnable_queue.task_count_hint(), contended_count, executing_queue_count, provisioning_tracker_count);
+                               assert!(maybe_checkpoint.is_none());
+                               maybe_checkpoint = Some(checkpoint);
                            },
-                           Err(crossbeam_channel::TryRecvError::Disconnected) => {
-                               assert_eq!(from_exec.len(), 0);
-                               from_exec_disconnected = true;
-                               info!("flushing1..: {:?} {} {} {} {}", (from_disconnected, from_exec_disconnected), channel_backed_runnable_queue.task_count_hint(), contended_count, executing_queue_count, provisioning_tracker_count);
-                               todo!(); //break;
+                           Err(_) => {
+                               assert_eq!(from_prev.len(), 0);
+                               assert!(!from_disconnected);
+                               assert!(maybe_checkpoint.is_none());
+                               from_disconnected = true;
+                               from_prev = never;
+                               trace!("flushing2..: {:?} {} {} {} {}", (from_disconnected, from_exec_disconnected), channel_backed_runnable_queue.task_count_hint(), contended_count, executing_queue_count, provisioning_tracker_count);
                            },
                        }
                    }
@@ -1835,12 +1808,10 @@ impl ScheduleStage {
                     );
                     if let Some(ee) = maybe_ee {
                         executing_queue_count = executing_queue_count.checked_add(1).unwrap();
-                        //info!("    sc send high begin");
                         to_high_execute_substage
                             .unwrap_or(to_execute_substage)
-                            .send(ExecutablePayload(SpinWaitable::Payload(ee)))
+                            .send(ExecutablePayload(ee))
                             .unwrap();
-                        //info!("    sc send high end");
                     }
                     debug!("schedule_once id_{:016x} [C] ch(prev: {}, exec: {}+{}|{}), r: {}, u/c: {}/{}, (imm+provi)/max: ({}+{})/{} s: {} l(s+f): {}+{}", random_id, (if from_disconnected { "-".to_string() } else { format!("{}", from_prev.len()) }), to_high_execute_substage.map(|t| format!("{}", t.len())).unwrap_or("-".into()), to_execute_substage.len(), from_exec.len(), channel_backed_runnable_queue.task_count_hint(), address_book.uncontended_task_ids.len(), contended_count, executing_queue_count, provisioning_tracker_count, max_executing_queue_count, address_book.stuck_tasks.len(), processed_count, failed_lock_count);
 
@@ -1886,9 +1857,7 @@ impl ScheduleStage {
                     );
                     if let Some(ee) = maybe_ee {
                         executing_queue_count = executing_queue_count.checked_add(1).unwrap();
-                        //info!("    sc send begin");
-                        to_execute_substage.send(ExecutablePayload(SpinWaitable::Payload(ee))).unwrap();
-                        //info!("    sc send end");
+                        to_execute_substage.send(ExecutablePayload(ee)).unwrap();
                     }
                     debug!("schedule_once id_{:016x} [R] ch(prev: {}, exec: {}+{}|{}), r: {}, u/c: {}/{}, (imm+provi)/max: ({}+{})/{} s: {} l(s+f): {}+{}", random_id, (if from_disconnected { "-".to_string() } else { format!("{}", from_prev.len()) }), to_high_execute_substage.map(|t| format!("{}", t.len())).unwrap_or("-".into()), to_execute_substage.len(), from_exec.len(), channel_backed_runnable_queue.task_count_hint(), address_book.uncontended_task_ids.len(), contended_count, executing_queue_count, provisioning_tracker_count, max_executing_queue_count, address_book.stuck_tasks.len(), processed_count, failed_lock_count);
 
@@ -1947,7 +1916,6 @@ impl ScheduleStage {
                         empty_from_exec = from_exec_len == 0;
                         executing_queue_count = executing_queue_count.checked_sub(1).unwrap();
                         processed_count = processed_count.checked_add(1).unwrap();
-                        //info!("    sc pending recv");
                         Self::commit_processed_execution(
                             ast,
                             &mut processed_execution_environment,
@@ -2033,8 +2001,8 @@ impl ScheduleStage {
         runnable_queue: &mut TaskQueue,
         address_book: &mut AddressBook,
         from: &crossbeam_channel::Receiver<SchedulablePayload<C>>,
-        to_execute_substage: &crossbeam_channel::Sender<ExecutablePayload<C>>,
-        to_high_execute_substage: Option<&crossbeam_channel::Sender<ExecutablePayload<C>>>,
+        to_execute_substage: &crossbeam_channel::Sender<ExecutablePayload>,
+        to_high_execute_substage: Option<&crossbeam_channel::Sender<ExecutablePayload>>,
         from_execute_substage: &crossbeam_channel::Receiver<UnlockablePayload<T>>,
         maybe_to_next_stage: Option<&crossbeam_channel::Sender<ExaminablePayload<T, C>>>, // assume nonblocking
     ) -> Option<std::sync::Arc<Checkpoint<C>>> {
@@ -2059,11 +2027,11 @@ impl ScheduleStage {
 }
 
 pub struct SchedulablePayload<C>(pub Flushable<TaskInQueue, C>);
-pub struct ExecutablePayload<C>(pub SpinWaitable<Box<ExecutionEnvironment>, C>);
+pub struct ExecutablePayload(pub Box<ExecutionEnvironment>);
 pub struct UnlockablePayload<T>(pub Box<ExecutionEnvironment>, pub T);
 pub struct ExaminablePayload<T, C>(pub Flushable<(Box<ExecutionEnvironment>, T), C>);
 
-pub struct Checkpoint<T>(std::sync::Mutex<(usize, Option<T>)>, std::sync::Condvar, bool);
+pub struct Checkpoint<T>(std::sync::Mutex<(usize, Option<T>)>, std::sync::Condvar);
 
 impl<T> Checkpoint<T> {
     pub fn wait_for_restart(&self, maybe_given_restart_value: Option<T>) {
@@ -2085,7 +2053,7 @@ impl<T> Checkpoint<T> {
         }
 
         if *self_remaining_threads == 0 {
-            assert!(self.2 || self_return_value.is_some());
+            assert!(self_return_value.is_some());
             drop(self_remaining_threads);
             self.1.notify_all();
             trace!(
@@ -2114,11 +2082,10 @@ impl<T> Checkpoint<T> {
         self_return_value.take().unwrap()
     }
 
-    pub fn new(remaining_threads: usize, no_restart_value: bool) -> std::sync::Arc<Self> {
+    pub fn new(remaining_threads: usize) -> std::sync::Arc<Self> {
         std::sync::Arc::new(Self(
             std::sync::Mutex::new((remaining_threads, None)),
             std::sync::Condvar::new(),
-            no_restart_value,
         ))
     }
 }
@@ -2127,9 +2094,3 @@ pub enum Flushable<T, C> {
     Payload(T),
     Flush(std::sync::Arc<Checkpoint<C>>),
 }
-
-pub enum SpinWaitable<T, C> {
-    Payload(T),
-    Spin,
-    Flush(std::sync::Arc<Checkpoint<C>>),
-}

From 7147e9e09acca8a35547446b4ccbeb6661f89093 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 9 Oct 2022 13:10:52 +0900
Subject: [PATCH 3193/3199] small test program...

---
 channel-latency/Cargo.toml  | 13 +++++++++++++
 channel-latency/src/main.rs | 39 +++++++++++++++++++++++++++++++++++++
 2 files changed, 52 insertions(+)
 create mode 100644 channel-latency/Cargo.toml
 create mode 100644 channel-latency/src/main.rs

diff --git a/channel-latency/Cargo.toml b/channel-latency/Cargo.toml
new file mode 100644
index 00000000000000..30ee293baf7aa0
--- /dev/null
+++ b/channel-latency/Cargo.toml
@@ -0,0 +1,13 @@
+[package]
+name = "channel-latency"
+version = "0.1.0"
+edition = "2021"
+
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+
+[dependencies]
+solana-logger = { path = "../logger", version = "=1.12.0" }
+log = "0.4.17"
+crossbeam-channel = "0.5"
+rand = "0.7.0"
+flume = "0.10.14"
diff --git a/channel-latency/src/main.rs b/channel-latency/src/main.rs
new file mode 100644
index 00000000000000..ecef5a632cd9b9
--- /dev/null
+++ b/channel-latency/src/main.rs
@@ -0,0 +1,39 @@
+use log::*;
+use rand::Rng;
+
+fn main() {
+    solana_logger::setup();
+
+    //let (s, r) = crossbeam_channel::unbounded();
+    let (s, r) = crossbeam_channel::bounded(100_000);
+    //let (s, r) = flume::unbounded();
+    //let (s, r) = flume::bounded(100_000);
+
+    std::thread::scope(|scope| {
+        std::thread::Builder::new().name("send".into()).spawn_scoped(scope, || {
+            let mut t = 0;
+
+            loop {
+                for i in 0..10_000_000 {
+                    t += rand::thread_rng().gen::<u64>();
+                }
+                info!("sent begin");
+                //s.send_buffered(()).unwrap();
+                s.send_buffered(()).unwrap();
+                info!("sent end");
+            }
+            info!("{}", t);
+        }).unwrap();
+
+        std::thread::Builder::new().name("recv".into()).spawn_scoped(scope, || {
+            loop {
+                /*if let Ok(u) = r.try_recv() {
+                    info!("recv");
+                }*/
+                if let Ok(u) = r.recv() {
+                    info!("recv");
+                }
+            }
+        }).unwrap();
+    });
+}

From bd9e113a00b4b0e87f140af0e24caf9e83c6f9d7 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 9 Oct 2022 15:28:58 +0900
Subject: [PATCH 3194/3199] small test program...

---
 Cargo.lock                  | 58 ++++++++++++++++++++++++++++++-------
 Cargo.toml                  |  1 +
 channel-latency/src/main.rs | 21 ++++++++------
 3 files changed, 60 insertions(+), 20 deletions(-)

diff --git a/Cargo.lock b/Cargo.lock
index 5e2d6216c21ff6..71516fe98960fb 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -60,7 +60,7 @@ version = "0.7.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "fcb51a0695d8f838b1ee009b3fbf66bda078cd64590202a864a8f3e8c4315c47"
 dependencies = [
- "getrandom 0.2.3",
+ "getrandom 0.2.7",
  "once_cell",
  "version_check",
 ]
@@ -333,7 +333,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "b62ddb9cb1ec0a098ad4bbf9344d0713fa193ae1a80af55febcff2627b6a00c1"
 dependencies = [
  "futures-core",
- "getrandom 0.2.3",
+ "getrandom 0.2.7",
  "instant",
  "pin-project-lite",
  "rand 0.8.5",
@@ -736,6 +736,17 @@ version = "1.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
 
+[[package]]
+name = "channel-latency"
+version = "0.1.0"
+dependencies = [
+ "crossbeam-channel",
+ "flume",
+ "log",
+ "rand 0.7.3",
+ "solana-logger 1.12.0",
+]
+
 [[package]]
 name = "chrono"
 version = "0.4.22"
@@ -1581,6 +1592,19 @@ dependencies = [
  "num-traits",
 ]
 
+[[package]]
+name = "flume"
+version = "0.10.14"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1657b4441c3403d9f7b3409e47575237dac27b1b5726df654a6ecbf92f0f7577"
+dependencies = [
+ "futures-core",
+ "futures-sink",
+ "nanorand",
+ "pin-project",
+ "spin 0.9.2",
+]
+
 [[package]]
 name = "fnv"
 version = "1.0.7"
@@ -1800,14 +1824,14 @@ dependencies = [
 
 [[package]]
 name = "getrandom"
-version = "0.2.3"
+version = "0.2.7"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7fcd999463524c52659517fe2cea98493cfe485d10565e7b0fb07dbba7ad2753"
+checksum = "4eb1a864a501629691edf6c15a593b7a51eebaa1e8468e9ddc623de7c9b58ec6"
 dependencies = [
  "cfg-if 1.0.0",
  "js-sys",
  "libc",
- "wasi 0.10.2+wasi-snapshot-preview1",
+ "wasi 0.11.0+wasi-snapshot-preview1",
  "wasm-bindgen",
 ]
 
@@ -2731,6 +2755,15 @@ version = "0.8.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "e5ce46fe64a9d73be07dcbe690a38ce1b293be448fd8ce1e6c1b8062c9f72c6a"
 
+[[package]]
+name = "nanorand"
+version = "0.7.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6a51313c5820b0b02bd422f4b44776fbf47961755c74ce64afc73bfad10226c3"
+dependencies = [
+ "getrandom 0.2.7",
+]
+
 [[package]]
 name = "native-tls"
 version = "0.2.8"
@@ -3760,7 +3793,7 @@ version = "0.6.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "d34f1408f55294453790c48b2f1ebbb1c5b4b7563eb1f418bcfcfdbb06ebb4e7"
 dependencies = [
- "getrandom 0.2.3",
+ "getrandom 0.2.7",
 ]
 
 [[package]]
@@ -3870,7 +3903,7 @@ version = "0.4.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "528532f3d801c87aec9def2add9ca802fe569e44a544afe633765267840abe64"
 dependencies = [
- "getrandom 0.2.3",
+ "getrandom 0.2.7",
  "redox_syscall",
 ]
 
@@ -5787,7 +5820,7 @@ dependencies = [
  "console_error_panic_hook",
  "console_log",
  "curve25519-dalek",
- "getrandom 0.2.3",
+ "getrandom 0.2.7",
  "itertools",
  "js-sys",
  "lazy_static",
@@ -5836,7 +5869,7 @@ dependencies = [
  "console_error_panic_hook",
  "console_log",
  "curve25519-dalek",
- "getrandom 0.2.3",
+ "getrandom 0.2.7",
  "itertools",
  "js-sys",
  "lazy_static",
@@ -6886,6 +6919,9 @@ name = "spin"
 version = "0.9.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "511254be0c5bcf062b019a6c89c01a664aa359ded62f78aa72c6fc137c0590e5"
+dependencies = [
+ "lock_api",
+]
 
 [[package]]
 name = "spki"
@@ -7977,9 +8013,9 @@ checksum = "cccddf32554fecc6acb585f82a32a72e28b48f8c4c1883ddfeeeaa96f7d8e519"
 
 [[package]]
 name = "wasi"
-version = "0.10.2+wasi-snapshot-preview1"
+version = "0.11.0+wasi-snapshot-preview1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fd6fbd9a79829dd1ad0cc20627bf1ed606756a7f77edff7b66b7064f9cb327c6"
+checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423"
 
 [[package]]
 name = "wasm-bindgen"
diff --git a/Cargo.toml b/Cargo.toml
index c0d5d07c388068..6b152d554201b5 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -11,6 +11,7 @@ members = [
     "bench-tps",
     "bloom",
     "bucket_map",
+    "channel-latency",
     "clap-utils",
     "clap-v3-utils",
     "cli",
diff --git a/channel-latency/src/main.rs b/channel-latency/src/main.rs
index ecef5a632cd9b9..a295b7aa7a1dc1 100644
--- a/channel-latency/src/main.rs
+++ b/channel-latency/src/main.rs
@@ -4,8 +4,8 @@ use rand::Rng;
 fn main() {
     solana_logger::setup();
 
-    //let (s, r) = crossbeam_channel::unbounded();
-    let (s, r) = crossbeam_channel::bounded(100_000);
+    let (s, r) = crossbeam_channel::unbounded();
+    //let (s, r) = crossbeam_channel::bounded(100);
     //let (s, r) = flume::unbounded();
     //let (s, r) = flume::bounded(100_000);
 
@@ -17,21 +17,24 @@ fn main() {
                 for i in 0..10_000_000 {
                     t += rand::thread_rng().gen::<u64>();
                 }
-                info!("sent begin");
-                //s.send_buffered(()).unwrap();
-                s.send_buffered(()).unwrap();
-                info!("sent end");
+                //let msg = Box::new([0x33_u8; 100_000]); // ();
+                //let msg = [0x33_u8; 100]; // ();
+                let msg = ();
+
+                let a = std::time::Instant::now();
+                //info!("sent begin");
+                s.send_buffered(msg).unwrap();
+                //s.send_buffered(msg).unwrap();
+                info!("sent took: {:?}", a.elapsed());
             }
             info!("{}", t);
         }).unwrap();
 
         std::thread::Builder::new().name("recv".into()).spawn_scoped(scope, || {
             loop {
-                /*if let Ok(u) = r.try_recv() {
-                    info!("recv");
-                }*/
                 if let Ok(u) = r.recv() {
                     info!("recv");
+                    info!("");
                 }
             }
         }).unwrap();

From 933b1b8aa95488fd8e2d2da096f451fb6dc31371 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Wed, 2 Nov 2022 09:48:04 +0900
Subject: [PATCH 3195/3199] Salvage account_locks code from unified scheduler

---
 Cargo.lock                       | 65 ++++++++++++++++++++++++++++++++
 runtime/Cargo.toml               |  1 +
 runtime/src/bank.rs              |  5 ++-
 sdk/Cargo.toml                   |  1 +
 sdk/src/transaction/sanitized.rs |  7 +++-
 5 files changed, 77 insertions(+), 2 deletions(-)

diff --git a/Cargo.lock b/Cargo.lock
index 71516fe98960fb..a3d6307553761f 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -1143,6 +1143,41 @@ dependencies = [
  "zeroize",
 ]
 
+[[package]]
+name = "darling"
+version = "0.13.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a01d95850c592940db9b8194bc39f4bc0e89dee5c4265e4b1807c34a9aba453c"
+dependencies = [
+ "darling_core",
+ "darling_macro",
+]
+
+[[package]]
+name = "darling_core"
+version = "0.13.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "859d65a907b6852c9361e3185c862aae7fafd2887876799fa55f5f99dc40d610"
+dependencies = [
+ "fnv",
+ "ident_case",
+ "proc-macro2 1.0.41",
+ "quote 1.0.18",
+ "strsim 0.10.0",
+ "syn 1.0.98",
+]
+
+[[package]]
+name = "darling_macro"
+version = "0.13.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9c972679f83bdf9c42bd905396b6c3588a843a17f0f16dfcfa3e2c5d57441835"
+dependencies = [
+ "darling_core",
+ "quote 1.0.18",
+ "syn 1.0.98",
+]
+
 [[package]]
 name = "dashmap"
 version = "4.0.2"
@@ -2171,6 +2206,12 @@ dependencies = [
  "winapi 0.3.9",
 ]
 
+[[package]]
+name = "ident_case"
+version = "1.0.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39"
+
 [[package]]
 name = "idna"
 version = "0.1.5"
@@ -4333,6 +4374,28 @@ dependencies = [
  "serde",
 ]
 
+[[package]]
+name = "serde_with"
+version = "1.14.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "678b5a069e50bf00ecd22d0cd8ddf7c236f68581b03db652061ed5eb13a312ff"
+dependencies = [
+ "serde",
+ "serde_with_macros",
+]
+
+[[package]]
+name = "serde_with_macros"
+version = "1.5.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e182d6ec6f05393cc0e5ed1bf81ad6db3a8feedf8ee515ecdd369809bcce8082"
+dependencies = [
+ "darling",
+ "proc-macro2 1.0.41",
+ "quote 1.0.18",
+ "syn 1.0.98",
+]
+
 [[package]]
 name = "serde_yaml"
 version = "0.8.26"
@@ -6186,6 +6249,7 @@ dependencies = [
  "rustc_version 0.4.0",
  "serde",
  "serde_derive",
+ "serde_json",
  "solana-address-lookup-table-program",
  "solana-bucket-map",
  "solana-compute-budget-program",
@@ -6321,6 +6385,7 @@ dependencies = [
  "serde_bytes",
  "serde_derive",
  "serde_json",
+ "serde_with",
  "sha2 0.10.2",
  "sha3 0.10.2",
  "solana-frozen-abi 1.12.0",
diff --git a/runtime/Cargo.toml b/runtime/Cargo.toml
index 1bd2024e0d292a..6a10da51470919 100644
--- a/runtime/Cargo.toml
+++ b/runtime/Cargo.toml
@@ -41,6 +41,7 @@ rayon = "1.5.3"
 regex = "1.5.6"
 serde = { version = "1.0.143", features = ["rc"] }
 serde_derive = "1.0.103"
+serde_json = "1.0.81"
 solana-address-lookup-table-program = { path = "../programs/address-lookup-table", version = "=1.12.0" }
 solana-bucket-map = { path = "../bucket_map", version = "=1.12.0" }
 solana-compute-budget-program = { path = "../programs/compute-budget", version = "=1.12.0" }
diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 45ed9ad91c8145..7387d5f8d3b875 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1423,6 +1423,8 @@ impl Scheduler<ExecuteTimings> {
                 loop {
                 while let Ok(r) = retired_ee_receiver.recv_timeout(std::time::Duration::from_millis(20))
                 {
+                    use crate::transaction_priority_details::GetTransactionPriorityDetails;
+
                     match r {
                         solana_scheduler::ExaminablePayload(solana_scheduler::Flushable::Payload((mut ee, timings))) => {
                             cumulative_timings.accumulate(&timings);
@@ -1436,7 +1438,7 @@ impl Scheduler<ExecuteTimings> {
                                     ("index", ee.transaction_index, i64),
                                     ("thread", format!("solScExLane{:02}", ee.thx), String),
                                     ("signature", &sig, String),
-                                    ("account_locks_in_json", "{}", String),
+                                    ("account_locks_in_json", serde_json::to_string(&ee.task.tx.0.get_account_locks_unchecked()).unwrap(), String),
                                     (
                                         "status",
                                         format!("{:?}", ee.execution_result.as_ref().unwrap()),
@@ -1445,6 +1447,7 @@ impl Scheduler<ExecuteTimings> {
                                     ("duration", ee.execution_us, i64),
                                     ("cpu_duration", ee.execution_cpu_us, i64),
                                     ("compute_units", ee.cu, i64),
+                                    ("priority", ee.task.tx.0.get_transaction_priority_details().map(|d| d.priority).unwrap_or_default(), i64),
                                 );
                                 info!("execute_substage: slot: {} transaction_index: {} timings: {:?}", ee.slot, ee.transaction_index, timings);
                             }
diff --git a/sdk/Cargo.toml b/sdk/Cargo.toml
index 8956d98a35e472..94281903b34172 100644
--- a/sdk/Cargo.toml
+++ b/sdk/Cargo.toml
@@ -69,6 +69,7 @@ serde = "1.0.143"
 serde_bytes = "0.11"
 serde_derive = "1.0.103"
 serde_json = { version = "1.0.83", optional = true }
+serde_with = "1.14.0"
 sha2 = "0.10.2"
 sha3 = { version = "0.10.2", optional = true }
 solana-frozen-abi = { path = "../frozen-abi", version = "=1.12.0" }
diff --git a/sdk/src/transaction/sanitized.rs b/sdk/src/transaction/sanitized.rs
index 7d1b34a4ab41e6..5cc428321e9fc1 100644
--- a/sdk/src/transaction/sanitized.rs
+++ b/sdk/src/transaction/sanitized.rs
@@ -34,12 +34,17 @@ pub struct SanitizedTransaction {
     signatures: Vec<Signature>,
 }
 
+use serde_with::*;
+
 /// Set of accounts that must be locked for safe transaction processing
-#[derive(Debug, Clone, Default)]
+#[serde_as]
+#[derive(Debug, Clone, Default, Serialize)]
 pub struct TransactionAccountLocks<'a> {
     /// List of readonly account key locks
+    #[serde_as(as = "Vec<DisplayFromStr>")]
     pub readonly: Vec<&'a Pubkey>,
     /// List of writable account key locks
+    #[serde_as(as = "Vec<DisplayFromStr>")]
     pub writable: Vec<&'a Pubkey>,
 }
 

From cfe29ad7407441688bd45d9136971922ffeeaa21 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Wed, 2 Nov 2022 09:57:34 +0900
Subject: [PATCH 3196/3199] Consistent env...

---
 ledger-tool/src/main.rs | 2 +-
 runtime/src/bank.rs     | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/ledger-tool/src/main.rs b/ledger-tool/src/main.rs
index 57bda5bf6d2088..5ceee33be4bace 100644
--- a/ledger-tool/src/main.rs
+++ b/ledger-tool/src/main.rs
@@ -313,7 +313,7 @@ fn output_slot(
                 .spawn(move || {
                     use solana_metrics::datapoint_info;
                     let current_thread_name = std::thread::current().name().unwrap().to_string();
-                    let send_metrics = std::env::var("SEND_METRICS").is_ok();
+                    let send_metrics = std::env::var("SOLANA_TRANSACTION_TIMINGS").is_ok();
 
                     for step in 0.. {
                         let mut ee = pre_execute_env_receiver.recv().unwrap().0;
diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
index 7387d5f8d3b875..4c32e0f8ec2e0b 100644
--- a/runtime/src/bank.rs
+++ b/runtime/src/bank.rs
@@ -1321,7 +1321,7 @@ impl Scheduler<ExecuteTimings> {
             .parse::<usize>()
             .unwrap();
 
-        let send_metrics = std::env::var("SEND_METRICS").is_ok();
+        let send_metrics = std::env::var("SOLANA_TRANSACTION_TIMINGS").is_ok();
 
         let max_thread_priority = std::env::var("MAX_THREAD_PRIORITY").is_ok();
 

From 639502f35ad5acf3a1311231407231d480c1674e Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 5 Nov 2022 11:59:35 +0900
Subject: [PATCH 3197/3199] More aggressive read-uncontended same-locking

---
 Cargo.lock           |  7 +++++++
 scheduler/Cargo.toml |  1 +
 scheduler/src/lib.rs | 50 ++++++++++++++++++++++++++++----------------
 3 files changed, 40 insertions(+), 18 deletions(-)

diff --git a/Cargo.lock b/Cargo.lock
index a3d6307553761f..9a6b3f0a503eb2 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -576,6 +576,12 @@ dependencies = [
  "serde",
 ]
 
+[[package]]
+name = "by_address"
+version = "1.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "bf8dba2868114ed769a1f2590fc9ae5eb331175b44313b6c9b922f8f7ca813d0"
+
 [[package]]
 name = "byte-tools"
 version = "0.3.1"
@@ -6281,6 +6287,7 @@ dependencies = [
 name = "solana-scheduler"
 version = "1.12.0"
 dependencies = [
+ "by_address",
  "crossbeam-channel",
  "dashmap 4.0.2",
  "log",
diff --git a/scheduler/Cargo.toml b/scheduler/Cargo.toml
index cd45643a28d800..971fd63785f8c5 100644
--- a/scheduler/Cargo.toml
+++ b/scheduler/Cargo.toml
@@ -11,6 +11,7 @@ documentation = "https://docs.rs/solana-scheduler"
 publish = true
 
 [dependencies]
+by_address = "1.1.0"
 crossbeam-channel = "0.5.5"
 dashmap = { version = "4.0.2" }
 log = "0.4.17"
diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 1bf1343aaac06c..48964877005bfb 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -25,8 +25,8 @@ type PageRcInner = triomphe::Arc<(
     std::sync::atomic::AtomicUsize,
 )>;
 
-#[derive(Debug, Clone)]
-pub struct PageRc(PageRcInner);
+#[derive(Debug, Clone, Hash, Eq, PartialEq)]
+pub struct PageRc(by_address::ByAddress<PageRcInner>);
 unsafe impl Send for PageRc {}
 unsafe impl Sync for PageRc {}
 
@@ -108,7 +108,7 @@ pub unsafe trait NotAtScheduleThread: Copy {}
 
 impl PageRc {
     fn page_mut<AST: AtScheduleThread>(&self, _ast: AST) -> std::cell::RefMut<'_, Page> {
-        self.0 .0.borrow_mut()
+        self.0.0 .0.borrow_mut()
     }
 }
 
@@ -350,6 +350,7 @@ impl Page {
 type AddressMap = std::sync::Arc<dashmap::DashMap<Pubkey, PageRc>>;
 type TaskId = UniqueWeight;
 type WeightedTaskIds = std::collections::BTreeMap<TaskId, TaskInQueue>;
+type WeightedTaskIds2 = std::collections::BTreeMap<TaskId, (TaskInQueue, std::collections::HashSet<PageRc>)>;
 //type AddressMapEntry<'a, K, V> = std::collections::hash_map::Entry<'a, K, V>;
 type AddressMapEntry<'a> = dashmap::mapref::entry::Entry<'a, Pubkey, PageRc>;
 
@@ -359,7 +360,7 @@ type StuckTaskId = (CU, TaskId);
 #[derive(Default)]
 pub struct AddressBook {
     book: AddressMap,
-    uncontended_task_ids: WeightedTaskIds,
+    uncontended_task_ids: WeightedTaskIds2,
     fulfilled_provisional_task_ids: WeightedTaskIds,
     stuck_tasks: std::collections::BTreeMap<StuckTaskId, TaskInQueue>,
 }
@@ -586,11 +587,11 @@ impl Preloader {
     #[inline(never)]
     pub fn load(&self, address: Pubkey) -> PageRc {
         PageRc::clone(&self.book.entry(address).or_insert_with(|| {
-            PageRc(PageRcInner::new((
+            PageRc(by_address::ByAddress(PageRcInner::new((
                 core::cell::RefCell::new(Page::new(&address, Usage::unused())),
                 //Default::default(),
                 Default::default(),
-            )))
+            ))))
         }))
     }
 }
@@ -1039,10 +1040,9 @@ pub fn get_transaction_priority_details(tx: &SanitizedTransaction) -> u64 {
 
 pub struct ScheduleStage {}
 
-#[derive(PartialEq, Eq)]
 enum TaskSource {
     Runnable,
-    Contended,
+    Contended(std::collections::HashSet<PageRc>),
     Stuck,
 }
 
@@ -1075,7 +1075,7 @@ impl ScheduleStage {
     #[inline(never)]
     fn get_heaviest_from_contended<'a>(
         address_book: &'a mut AddressBook,
-    ) -> Option<std::collections::btree_map::OccupiedEntry<'a, UniqueWeight, TaskInQueue>> {
+    ) -> Option<std::collections::btree_map::OccupiedEntry<'a, UniqueWeight, (TaskInQueue, std::collections::HashSet<PageRc>)>> {
         address_book.uncontended_task_ids.last_entry()
     }
 
@@ -1110,7 +1110,7 @@ impl ScheduleStage {
                     None
                 } else {
                     let t = weight_from_contended.remove();
-                    Some((TaskSource::Contended, t))
+                    Some((TaskSource::Contended(t.1), t.0))
                 }
             }
             (Some(heaviest_runnable_entry), Some(weight_from_contended)) => {
@@ -1138,7 +1138,7 @@ impl ScheduleStage {
                     } else {
                         trace!("select: contended > runnnable, !runnable_exclusive)");
                         let t = weight_from_contended.remove();
-                        Some((TaskSource::Contended, t))
+                        Some((TaskSource::Contended(t.1), t.0))
                     }
                 } else {
                     unreachable!(
@@ -1203,7 +1203,7 @@ impl ScheduleStage {
                 contended_count,
                 task_selection,
             ) {
-                let from_runnable = task_source == TaskSource::Runnable;
+                let from_runnable = matches!(task_source, TaskSource::Runnable);
                 if from_runnable {
                     next_task.record_queue_time(*sequence_clock, *queue_clock);
                     *queue_clock = queue_clock.checked_add(1).unwrap();
@@ -1270,7 +1270,7 @@ impl ScheduleStage {
                         //address_book.uncontended_task_ids.clear();
                     }
 
-                    if from_runnable || task_source == TaskSource::Stuck {
+                    if from_runnable || matches!(task_source, TaskSource::Stuck) {
                         // for the case of being struck, we have already removed it from
                         // stuck_tasks, so pretend to add anew one.
                         // todo: optimize this needless operation
@@ -1285,7 +1285,7 @@ impl ScheduleStage {
                         if from_runnable {
                             // continue; // continue to prefer depleting the possibly-non-empty runnable queue
                             break;
-                        } else if task_source == TaskSource::Stuck {
+                        } else if matches!(task_source, TaskSource::Stuck) {
                             // need to bail out immediately to avoid going to infinite loop of re-processing
                             // the struck task again.
                             // todo?: buffer restuck tasks until readd to the stuck tasks until
@@ -1295,7 +1295,7 @@ impl ScheduleStage {
                         } else {
                             unreachable!();
                         }
-                    } else if task_source == TaskSource::Contended {
+                    } else if matches!(task_source, TaskSource::Contended(_)) {
                         // todo: remove this task from stuck_tasks before update_busiest_page_cu
                         /*
                         let removed = address_book
@@ -1354,6 +1354,18 @@ impl ScheduleStage {
                 if !from_runnable {
                     *contended_count = contended_count.checked_sub(1).unwrap();
                     next_task.mark_as_uncontended();
+                    if let TaskSource::Contended(uncontendeds) = task_source {
+                        for lock_attempt in next_task.lock_attempts_mut(ast).iter().filter(|l| l.requested_usage == RequestedUsage::Readonly && uncontendeds.contains(&l.target)) {
+                            if let Some(task) = lock_attempt.target_page_mut(ast).task_ids.reindex(false, &unique_weight) {
+                                if task.currently_contended() {
+                                    let uti = address_book
+                                        .uncontended_task_ids
+                                        .entry(task.unique_weight).or_insert((task, Default::default()));
+                                    uti.1.insert(lock_attempt.target.clone());
+                                }
+                            }
+                        }
+                    }
                 } else {
                     next_task.update_busiest_page_cu(busiest_page_cu);
                 }
@@ -1428,9 +1440,10 @@ impl ScheduleStage {
                     task.currently_contended() {
                         //assert!(task.currently_contended());
                         //inserted = true;
-                        address_book
+                        let uti = address_book
                             .uncontended_task_ids
-                            .insert(task.unique_weight, task);
+                            .entry(task.unique_weight).or_insert((task, Default::default()));
+                        uti.1.insert(l.target.clone());
                     } /*else {
                           let contended_unique_weights = &page.contended_unique_weights;
                           contended_unique_weights.heaviest_task_cursor().map(|mut task_cursor| {
@@ -1450,7 +1463,8 @@ impl ScheduleStage {
                               }
                               found.then(|| Task::clone_in_queue(task))
                           }).flatten().map(|task| {
-                              address_book.uncontended_task_ids.insert(task.unique_weight, task);
+                              let uti = address_book.uncontended_task_ids.entry(task.unique_weight).or_insert(task);
+                              uti.1.insert(????);
                               ()
                           });
                       }*/

From c7576288eb3749a46e70bde1315a069f3ba1a7bc Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sat, 5 Nov 2022 21:06:59 +0900
Subject: [PATCH 3198/3199] Relax read-aggressive-lock

---
 scheduler/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 48964877005bfb..069948e592f1c6 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -1355,7 +1355,7 @@ impl ScheduleStage {
                     *contended_count = contended_count.checked_sub(1).unwrap();
                     next_task.mark_as_uncontended();
                     if let TaskSource::Contended(uncontendeds) = task_source {
-                        for lock_attempt in next_task.lock_attempts_mut(ast).iter().filter(|l| l.requested_usage == RequestedUsage::Readonly && uncontendeds.contains(&l.target)) {
+                        for lock_attempt in next_task.lock_attempts_mut(ast).iter().filter(|l| l.requested_usage == RequestedUsage::Readonly /*&& uncontendeds.contains(&l.target)*/) {
                             if let Some(task) = lock_attempt.target_page_mut(ast).task_ids.reindex(false, &unique_weight) {
                                 if task.currently_contended() {
                                     let uti = address_book

From 7f048c37a7028c08419ee6ddf4ce56523330adb3 Mon Sep 17 00:00:00 2001
From: Ryo Onodera <ryoqun@gmail.com>
Date: Sun, 6 Nov 2022 11:00:32 +0900
Subject: [PATCH 3199/3199] reRelax read-aggressive-lock

---
 scheduler/src/lib.rs | 28 +++++++++++++++++++++-------
 1 file changed, 21 insertions(+), 7 deletions(-)

diff --git a/scheduler/src/lib.rs b/scheduler/src/lib.rs
index 069948e592f1c6..59e6eb43a09b66 100644
--- a/scheduler/src/lib.rs
+++ b/scheduler/src/lib.rs
@@ -83,9 +83,10 @@ impl ExecutionEnvironment {
             };
 
             if should_remove && lock_attempt.requested_usage == RequestedUsage::Writable {
-                lock_attempt
-                    .target_contended_write_task_count()
-                    .fetch_sub(1, std::sync::atomic::Ordering::SeqCst);
+                //lock_attempt
+                //    .target_contended_write_task_count()
+                //    .fetch_sub(1, std::sync::atomic::Ordering::SeqCst);
+                lock_attempt.target_page_mut(ast).write_task_ids.remove(&uq);
             }
         }
     }
@@ -154,12 +155,19 @@ impl LockAttempt {
     pub fn target_contended_unique_weights(&self) -> &TaskIds {
         panic!()//&self.target.0 .1
     }
-    */
 
     pub fn target_contended_write_task_count(&self) -> &std::sync::atomic::AtomicUsize {
         &self.target.0 .1
     }
 
+    */
+
+    /*
+    pub fn remember_write_task_id<AST: AtScheduleThread>(&self, kst: AST, unique_weight: UniqueWeight) {
+        self.target_page_mut(ast).write_task_ids.insert(unique_weight);
+    }
+    */
+
     fn target_page_mut<AST: AtScheduleThread>(&self, ast: AST) -> std::cell::RefMut<'_, Page> {
         self.target.page_mut(ast)
     }
@@ -323,6 +331,7 @@ pub struct Page {
     provisional_task_ids: Vec<triomphe::Arc<ProvisioningTracker>>,
     cu: CU,
     task_ids: TaskIds,
+    write_task_ids: std::collections::BTreeSet<UniqueWeight>,
     //loaded account from Accounts db
     //comulative_cu for qos; i.e. track serialized cumulative keyed by addresses and bail out block
     //producing as soon as any one of cu from the executing thread reaches to the limit
@@ -337,6 +346,7 @@ impl Page {
             provisional_task_ids: Default::default(),
             cu: Default::default(),
             task_ids: Default::default(),
+            write_task_ids: Default::default(),
         }
     }
 
@@ -418,10 +428,13 @@ impl AddressBook {
         } else if tcuw.unwrap() == *unique_weight {
             true
         } else if attempt.requested_usage == RequestedUsage::Readonly
+            /*
             && attempt
                 .target_contended_write_task_count()
                 .load(std::sync::atomic::Ordering::SeqCst)
                 == 0
+            */
+            && attempt.target_page_mut(ast).write_task_ids.last().map(|j| unique_weight > j).unwrap_or(true)
         {
             true
         } else {
@@ -840,9 +853,10 @@ impl Task {
                 .insert_task(this.unique_weight, Task::clone_in_queue(this));
 
             if lock_attempt.requested_usage == RequestedUsage::Writable {
-                lock_attempt
-                    .target_contended_write_task_count()
-                    .fetch_add(1, std::sync::atomic::Ordering::SeqCst);
+                //lock_attempt
+                //    .target_contended_write_task_count()
+                //    .fetch_add(1, std::sync::atomic::Ordering::SeqCst);
+                lock_attempt.target_page_mut(ast).write_task_ids.insert(this.unique_weight);
             }
         }
         //let a = Task::clone_in_queue(this);