From 5d5301b35131cafec2c5cdf3fb8ac01b23d0d473 Mon Sep 17 00:00:00 2001 From: Niko Matsakis Date: Sat, 2 Nov 2019 08:58:46 -0400 Subject: [PATCH 01/33] adjust sleep module API to have an `IdleState` object This makes the "idle state" private to the sleep module. --- rayon-core/src/registry.rs | 9 ++--- rayon-core/src/sleep/mod.rs | 66 +++++++++++++++++++++++++------------ 2 files changed, 50 insertions(+), 25 deletions(-) diff --git a/rayon-core/src/registry.rs b/rayon-core/src/registry.rs index 5ef77e09d..a03a782db 100644 --- a/rayon-core/src/registry.rs +++ b/rayon-core/src/registry.rs @@ -653,7 +653,7 @@ impl WorkerThread { // accesses, which would be *very bad* let abort_guard = unwind::AbortIfPanic; - let mut yields = 0; + let mut idle_state = self.registry.sleep.start_looking(self.index); while !latch.probe() { // Try to find some work to do. We give preference first // to things in our local deque, then in other workers @@ -665,17 +665,18 @@ impl WorkerThread { .or_else(|| self.steal()) .or_else(|| self.registry.pop_injected_job(self.index)) { - yields = self.registry.sleep.work_found(self.index, yields); + self.registry.sleep.work_found(idle_state); self.execute(job); + idle_state = self.registry.sleep.start_looking(self.index); } else { - yields = self.registry.sleep.no_work_found(self.index, yields); + self.registry.sleep.no_work_found(&mut idle_state); } } // If we were sleepy, we are not anymore. We "found work" -- // whatever the surrounding thread was doing before it had to // wait. - self.registry.sleep.work_found(self.index, yields); + self.registry.sleep.work_found(idle_state); log!(LatchSet { worker: self.index }); mem::forget(abort_guard); // successful execution, do not abort diff --git a/rayon-core/src/sleep/mod.rs b/rayon-core/src/sleep/mod.rs index 002dc483f..a464a05d1 100644 --- a/rayon-core/src/sleep/mod.rs +++ b/rayon-core/src/sleep/mod.rs @@ -13,6 +13,19 @@ pub(super) struct Sleep { tickle: Condvar, } +/// An instance of this struct is created when a thread becomes idle. +/// It is consumed when the thread finds work, and passed by `&mut` +/// reference for operations that preserve the idle state. (In other +/// words, producing one of these structs is evidence the thread is +/// idle.) It tracks state such as how long the thread has been idle. +pub(super) struct IdleState { + /// What is worker index of the idle thread? + worker_index: usize, + + /// How many rounds have we been circling without sleeping? + rounds: usize, +} + const AWAKE: usize = 0; const SLEEPING: usize = 1; @@ -46,50 +59,61 @@ impl Sleep { } #[inline] - pub(super) fn work_found(&self, worker_index: usize, yields: usize) -> usize { + pub(super) fn start_looking(&self, worker_index: usize) -> IdleState { + IdleState { + worker_index, + rounds: 0, + } + } + + #[inline] + pub(super) fn work_found(&self, idle_state: IdleState) { + let IdleState { worker_index, rounds } = idle_state; + log!(FoundWork { worker: worker_index, - yields, + yields: rounds, }); - if yields > ROUNDS_UNTIL_SLEEPY { + + if rounds > ROUNDS_UNTIL_SLEEPY { // FIXME tickling here is a bit extreme; mostly we want to "release the lock" // from us being sleepy, we don't necessarily need to wake others // who are sleeping self.tickle(worker_index); } - 0 } #[inline] - pub(super) fn no_work_found(&self, worker_index: usize, yields: usize) -> usize { + pub(super) fn no_work_found(&self, idle_state: &mut IdleState) { + let IdleState { worker_index, rounds } = idle_state; log!(DidNotFindWork { - worker: worker_index, - yields, + worker: *worker_index, + yields: *rounds, }); - if yields < ROUNDS_UNTIL_SLEEPY { + if *rounds < ROUNDS_UNTIL_SLEEPY { thread::yield_now(); - yields + 1 - } else if yields == ROUNDS_UNTIL_SLEEPY { + *rounds += 1; + } else if *rounds == ROUNDS_UNTIL_SLEEPY { thread::yield_now(); - if self.get_sleepy(worker_index) { - yields + 1 + if self.get_sleepy(*worker_index) { + *rounds += 1; } else { - yields + *rounds = 0; } - } else if yields < ROUNDS_UNTIL_ASLEEP { + } else if *rounds < ROUNDS_UNTIL_ASLEEP { thread::yield_now(); - if self.still_sleepy(worker_index) { - yields + 1 + if self.still_sleepy(*worker_index) { + *rounds += 1; } else { log!(GotInterrupted { - worker: worker_index + worker: *worker_index }); - 0 + *rounds = 0; } } else { - debug_assert_eq!(yields, ROUNDS_UNTIL_ASLEEP); - self.sleep(worker_index); - 0 + debug_assert_eq!(*rounds, ROUNDS_UNTIL_ASLEEP); + self.sleep(*worker_index); + *rounds = 0; } } From bb599702cbfdbd03a29962fc1f5cf0d3f3cec42a Mon Sep 17 00:00:00 2001 From: Niko Matsakis Date: Sat, 2 Nov 2019 08:59:49 -0400 Subject: [PATCH 02/33] port to the new log api This involves: * creating a `Logger` object (always) * using `logger.log(|| ...)` instead of a macro * if logs are not compiled in, this should be optimized away * the logger tracks the number of active threads in various states etc * some log event names were "normalized" to be more consistent --- rayon-core/Cargo.toml | 1 + rayon-core/src/join/mod.rs | 11 +- rayon-core/src/log.rs | 505 +++++++++++++++++++++++++++++------- rayon-core/src/registry.rs | 37 ++- rayon-core/src/scope/mod.rs | 10 +- rayon-core/src/sleep/mod.rs | 33 +-- 6 files changed, 466 insertions(+), 131 deletions(-) diff --git a/rayon-core/Cargo.toml b/rayon-core/Cargo.toml index 03cd56dbb..b731baeea 100644 --- a/rayon-core/Cargo.toml +++ b/rayon-core/Cargo.toml @@ -18,6 +18,7 @@ categories = ["concurrency"] [dependencies] num_cpus = "1.2" lazy_static = "1" +crossbeam-channel = "0.3.9" crossbeam-deque = "0.7.2" crossbeam-queue = "0.2" crossbeam-utils = "0.7" diff --git a/rayon-core/src/join/mod.rs b/rayon-core/src/join/mod.rs index 295f97bc2..37dbf3ea3 100644 --- a/rayon-core/src/join/mod.rs +++ b/rayon-core/src/join/mod.rs @@ -131,10 +131,6 @@ where } registry::in_worker(|worker_thread, injected| unsafe { - log!(Join { - worker: worker_thread.index() - }); - // Create virtual wrapper for task b; this all has to be // done here so that the stack frame can keep it all live // long enough. @@ -160,13 +156,13 @@ where // Found it! Let's run it. // // Note that this could panic, but it's ok if we unwind here. - log!(PoppedRhs { + worker_thread.log(|| JobPoppedRhs { worker: worker_thread.index() }); let result_b = job_b.run_inline(injected); return (result_a, result_b); } else { - log!(PoppedJob { + worker_thread.log(|| JobPopped { worker: worker_thread.index() }); worker_thread.execute(job); @@ -174,9 +170,6 @@ where } else { // Local deque is empty. Time to steal from other // threads. - log!(LostJob { - worker: worker_thread.index() - }); worker_thread.wait_until(&job_b.latch); debug_assert!(job_b.latch.probe()); break; diff --git a/rayon-core/src/log.rs b/rayon-core/src/log.rs index 687370d0b..a62a65172 100644 --- a/rayon-core/src/log.rs +++ b/rayon-core/src/log.rs @@ -1,116 +1,429 @@ //! Debug Logging //! -//! To use in a debug build, set the env var `RAYON_LOG=1`. In a -//! release build, logs are compiled out. You will have to change -//! `DUMP_LOGS` to be `true`. +//! To use in a debug build, set the env var `RAYON_RS_LOG=1`. In a +//! release build, logs are compiled out by default unless Rayon is built +//! with `--cfg rayon_rs_log` (try `RUSTFLAGS="--cfg rayon_rs_log"`). //! -//! **Old environment variable:** `RAYON_LOG` is a one-to-one -//! replacement of the now deprecated `RAYON_RS_LOG` environment -//! variable, which is still supported for backwards compatibility. +//! Note that logs are an internally debugging tool and their format +//! is considered unstable, as are the details of how to enable them. -#[cfg(debug_assertions)] +use crossbeam_channel::{self, Receiver, Sender}; +use std::collections::VecDeque; use std::env; +use std::fs::File; +use std::io::{self, BufWriter, Write}; -#[cfg_attr(debug_assertions, derive(Debug))] -#[cfg_attr(not(debug_assertions), allow(dead_code))] +/// True if logs are compiled in. +pub(super) const LOG_ENABLED: bool = cfg!(any(rayon_rs_log, debug_assertions)); + +#[derive(Copy, Clone, PartialOrd, Ord, PartialEq, Eq, Debug)] pub(super) enum Event { - Tickle { - worker: usize, - old_state: usize, - }, - GetSleepy { - worker: usize, - state: usize, - }, - GotSleepy { - worker: usize, - old_state: usize, - new_state: usize, - }, - GotAwoken { - worker: usize, - }, - FellAsleep { - worker: usize, - }, - GotInterrupted { - worker: usize, - }, - FoundWork { - worker: usize, - yields: usize, - }, - DidNotFindWork { - worker: usize, - yields: usize, - }, - StoleWork { - worker: usize, - victim: usize, - }, - UninjectedWork { - worker: usize, - }, - WaitUntil { - worker: usize, - }, - LatchSet { - worker: usize, - }, - InjectJobs { - count: usize, - }, - Join { - worker: usize, - }, - PoppedJob { - worker: usize, - }, - PoppedRhs { + /// Flushes events to disk, used to terminate benchmarking. + Flush, + + /// Indicates that a worker thread started execution. + ThreadStart { worker: usize, }, - LostJob { + + /// Indicates that a worker thread started execution. + ThreadTerminate { worker: usize, }, - JobCompletedOk { - owner_thread: usize, - }, - JobPanickedErrorStored { - owner_thread: usize, - }, - JobPanickedErrorNotStored { - owner_thread: usize, - }, - ScopeCompletePanicked { - owner_thread: usize, - }, - ScopeCompleteNoPanic { - owner_thread: usize, - }, + + /// Indicates that a worker thread became idle, waiting for a latch. + ThreadIdle { worker: usize }, + + /// Indicates that an idle worker thread found work to do, after + /// yield rounds. It should no longer be considered idle. + ThreadFoundWork { worker: usize, yields: usize }, + + /// Indicates that a worker blocked on a latch observed that it was set. + /// + /// Internal debugging event that does not affect the state + /// machine. + ThreadSawLatchSet { worker: usize }, + + /// Indicates that an idle worker thread searched for another round without finding work. + ThreadNoWork { worker: usize, yields: usize }, + + /// Indicates that an idle worker became the sleepy worker. `state` is our internal sleep + /// state value (for debugging). + ThreadSleepy { worker: usize, state: usize, new_state: usize }, + + /// Indicates that the thread's attempt to fall asleep by somebody + /// making us not the sleepy worker. + ThreadSleepInterrupted { worker: usize }, + + /// Indicates that an idle worker has gone to sleep. + ThreadSleeping { worker: usize }, + + /// Indicates that a sleeping worker has awoken. + ThreadAwoken { worker: usize }, + + /// The given worker has pushed a job to its local deque. + JobPushed { worker: usize }, + + /// The given worker has popped a job from its local deque. + JobPopped { worker: usize }, + + /// The given worker has popped a job from its local deque, and + /// the job was the RHS of the `join` we were blocked on. + /// + /// Identical to `JobPopped` but for debugging. + JobPoppedRhs { worker: usize }, + + /// The given worker has stolen a job from the deque of another. + JobStolen { worker: usize, victim: usize }, + + /// N jobs were injected into the global queue. + JobsInjected { count: usize }, + + /// A job was removed from the global queue. + JobUninjected { worker: usize }, + + /// Indicates that a job completed "ok" as part of a scope. + JobCompletedOk { owner_thread: usize }, + + /// Indicates that a job panicked as part of a scope, and the + /// error was stored for later. + /// + /// Useful for debugging. + JobPanickedErrorStored { owner_thread: usize }, + + /// Indicates that a job panicked as part of a scope, and the + /// error was discarded. + /// + /// Useful for debugging. + JobPanickedErrorNotStored { owner_thread: usize }, + + /// Indicates that a scope completed with a panic. + /// + /// Useful for debugging. + ScopeCompletePanicked { owner_thread: usize }, + + /// Indicates that a scope completed with a panic. + /// + /// Useful for debugging. + ScopeCompleteNoPanic { owner_thread: usize }, + + /// A "tickle" came through and there were sleepy/sleeping workers. + Tickle { worker: usize, old_state: usize }, } -#[cfg(debug_assertions)] -lazy_static::lazy_static! { - pub(super) static ref LOG_ENV: bool = - env::var("RAYON_LOG").is_ok() || env::var("RAYON_RS_LOG").is_ok(); +/// Handle to the logging thread, if any. You can use this to deliver +/// logs. You can also clone it freely. +#[derive(Clone)] +pub(super) struct Logger { + sender: Option>, } -#[cfg(debug_assertions)] -macro_rules! log { - ($event:expr) => { - if *$crate::log::LOG_ENV { - eprintln!("{:?}", $event); +impl Logger { + pub(super) fn new(num_workers: usize) -> Logger { + if !LOG_ENABLED { + return Self::disabled(); } - }; + + // format: + // + // tail: -- dumps the last 10,000 events + // profile: -- dumps every Nth event + // all: -- dumps every event to the file + let env_log = match env::var("RAYON_RS_LOG") { + Ok(s) => s, + Err(_) => return Self::disabled(), + }; + + let (sender, receiver) = crossbeam_channel::unbounded(); + + if env_log.starts_with("tail:") { + let filename = env_log["tail:".len()..].to_string(); + ::std::thread::spawn(move || { + Self::tail_logger_thread(num_workers, filename, 10_000, receiver) + }); + } else if env_log == "all" { + ::std::thread::spawn(move || { + Self::all_logger_thread(num_workers, receiver) + }); + } else if env_log.starts_with("profile:") { + let filename = env_log["profile:".len()..].to_string(); + ::std::thread::spawn(move || { + Self::profile_logger_thread(num_workers, filename, 10_000, receiver) + }); + } else { + panic!("RAYON_RS_LOG should be 'all', 'tail:', or 'profile:'"); + } + + return Logger { + sender: Some(sender), + }; + } + + fn disabled() -> Logger { + Logger { sender: None } + } + + #[inline] + pub(super) fn log(&self, event: impl FnOnce() -> Event) { + if !LOG_ENABLED { + return; + } + + if let Some(sender) = &self.sender { + sender + .send(event()) + .unwrap(); + } + } + + fn profile_logger_thread( + num_workers: usize, + log_filename: String, + capacity: usize, + receiver: Receiver, + ) { + let file = File::create(&log_filename) + .unwrap_or_else(|err| panic!("failed to open `{}`: {}", log_filename, err)); + + let mut writer = BufWriter::new(file); + let mut events = Vec::with_capacity(capacity); + let mut state = SimulatorState::new(num_workers); + let timeout = std::time::Duration::from_secs(30); + + loop { + loop { + match receiver.recv_timeout(timeout) { + Ok(event) => { + if let Event::Flush = event { + break; + } else { + events.push(event); + } + } + + Err(_) => break, + } + + if events.len() == capacity { + break; + } + } + + for event in events.drain(..) { + if state.simulate(&event) { + state.dump(&mut writer, &event).unwrap(); + } + } + + writer.flush().unwrap(); + } + } + + fn tail_logger_thread( + num_workers: usize, + log_filename: String, + capacity: usize, + receiver: Receiver, + ) { + let file = File::create(&log_filename) + .unwrap_or_else(|err| panic!("failed to open `{}`: {}", log_filename, err)); + + let mut writer = BufWriter::new(file); + let mut events: VecDeque = VecDeque::with_capacity(capacity); + let mut state = SimulatorState::new(num_workers); + let timeout = std::time::Duration::from_secs(30); + let mut skipped = false; + + loop { + loop { + match receiver.recv_timeout(timeout) { + Ok(event) => { + if let Event::Flush = event { + // We ignore Flush events in tail mode -- + // we're really just looking for + // deadlocks. + continue; + } else { + if events.len() == capacity { + let event = events.pop_front().unwrap(); + state.simulate(&event); + skipped = true; + } + + events.push_back(event); + } + } + + Err(_) => break, + } + } + + if skipped { + write!(writer, "...\n").unwrap(); + skipped = false; + } + + for event in events.drain(..) { + // In tail mode, we dump *all* events out, whether or + // not they were 'interesting' to the state machine. + state.simulate(&event); + state.dump(&mut writer, &event).unwrap(); + } + + writer.flush().unwrap(); + } + } + + fn all_logger_thread( + num_workers: usize, + receiver: Receiver, + ) { + let stderr = std::io::stderr(); + let mut state = SimulatorState::new(num_workers); + + for event in receiver { + let mut writer = BufWriter::new(stderr.lock()); + state.simulate(&event); + state.dump(&mut writer, &event).unwrap(); + writer.flush().unwrap(); + } + } +} + +#[derive(Copy, Clone, PartialOrd, Ord, PartialEq, Eq, Debug)] +enum State { + Working, + Idle, + Sleeping, + Terminated, +} + +impl State { + fn letter(&self) -> char { + match self { + State::Working => 'W', + State::Idle => 'I', + + State::Sleeping => 'S', + State::Terminated => 'T', + } + } +} + +struct SimulatorState { + local_queue_size: Vec, + thread_states: Vec, + injector_size: usize, } -#[cfg(not(debug_assertions))] -macro_rules! log { - ($event:expr) => { - if false { - // Expand `$event` so it still appears used, but without - // any of the formatting code to be optimized away. - $event; +impl SimulatorState { + fn new(num_workers: usize) -> Self { + Self { + local_queue_size: (0..num_workers).map(|_| 0).collect(), + thread_states: (0..num_workers).map(|_| State::Working).collect(), + injector_size: 0, } - }; + } + + fn simulate(&mut self, event: &Event) -> bool { + match *event { + Event::ThreadIdle { worker, .. } => { + assert_eq!(self.thread_states[worker], State::Working); + self.thread_states[worker] = State::Idle; + true + } + + Event::ThreadStart { worker, .. } | Event::ThreadFoundWork { worker, .. } => { + self.thread_states[worker] = State::Working; + true + } + + Event::ThreadTerminate { worker, .. } => { + self.thread_states[worker] = State::Terminated; + true + } + + Event::ThreadSleeping { worker, .. } => { + assert_eq!(self.thread_states[worker], State::Idle); + self.thread_states[worker] = State::Sleeping; + true + } + + Event::ThreadAwoken { worker, .. } => { + assert_eq!(self.thread_states[worker], State::Sleeping); + self.thread_states[worker] = State::Idle; + true + } + + Event::JobPushed { worker } => { + self.local_queue_size[worker] += 1; + true + } + + Event::JobPopped { worker } | Event::JobPoppedRhs { worker } => { + self.local_queue_size[worker] -= 1; + true + } + + Event::JobStolen { victim, .. } => { + self.local_queue_size[victim] -= 1; + true + } + + Event::JobsInjected { count } => { + self.injector_size += count; + true + } + + Event::JobUninjected { .. } => { + self.injector_size -= 1; + true + } + + // remaining events are no-ops from pov of simulating the + // thread state + _ => false, + } + } + + fn dump(&mut self, w: &mut impl Write, event: &Event) -> io::Result<()> { + let num_idle_threads = self + .thread_states + .iter() + .filter(|s| **s == State::Idle) + .count(); + + let num_sleeping_threads = self + .thread_states + .iter() + .filter(|s| **s == State::Sleeping) + .count(); + + // we don't track this on rayon master, but other branches do + let num_notified_threads = 0; + + let num_pending_jobs: usize = self.local_queue_size.iter().sum(); + + write!(w, "{:2},", num_idle_threads)?; + write!(w, "{:2},", num_sleeping_threads)?; + write!(w, "{:2},", num_notified_threads)?; + write!(w, "{:4},", num_pending_jobs)?; + write!(w, "{:4},", self.injector_size)?; + + let event_str = format!("{:?}", event); + write!(w, r#""{:60}","#, event_str)?; + + for ((i, state), queue_size) in (0..).zip(&self.thread_states).zip(&self.local_queue_size) { + write!(w, " T{:02},{}", i, state.letter(),)?; + + if *queue_size > 0 { + write!(w, ",{:03},", queue_size)?; + } else { + write!(w, ", ,")?; + } + } + + write!(w, "\n")?; + Ok(()) + } } diff --git a/rayon-core/src/registry.rs b/rayon-core/src/registry.rs index a03a782db..9abe27baf 100644 --- a/rayon-core/src/registry.rs +++ b/rayon-core/src/registry.rs @@ -1,5 +1,6 @@ use crate::job::{JobFifo, JobRef, StackJob}; use crate::latch::{CountLatch, Latch, LatchProbe, LockLatch, SpinLatch, TickleLatch}; +use crate::log::Logger; use crate::log::Event::*; use crate::sleep::Sleep; use crate::unwind; @@ -132,6 +133,7 @@ where } pub(super) struct Registry { + logger: Logger, thread_infos: Vec, sleep: Sleep, injected_jobs: SegQueue, @@ -233,9 +235,11 @@ impl Registry { }) .unzip(); + let logger = Logger::new(n_threads); let registry = Arc::new(Registry { + logger: logger.clone(), thread_infos: stealers.into_iter().map(ThreadInfo::new).collect(), - sleep: Sleep::new(), + sleep: Sleep::new(logger), injected_jobs: SegQueue::new(), terminate_latch: CountLatch::new(), panic_handler: builder.take_panic_handler(), @@ -311,6 +315,11 @@ impl Registry { } } + #[inline] + pub(super) fn log(&self, event: impl FnOnce() -> crate::log::Event) { + self.logger.log(event) + } + pub(super) fn num_threads(&self) -> usize { self.thread_infos.len() } @@ -374,7 +383,7 @@ impl Registry { /// whatever worker has nothing to do. Use this is you know that /// you are not on a worker of this registry. pub(super) fn inject(&self, injected_jobs: &[JobRef]) { - log!(InjectJobs { + self.log(|| JobsInjected { count: injected_jobs.len() }); @@ -397,7 +406,7 @@ impl Registry { fn pop_injected_job(&self, worker_index: usize) -> Option { let job = self.injected_jobs.pop().ok(); if job.is_some() { - log!(UninjectedWork { + self.log(|| JobUninjected { worker: worker_index }); } @@ -450,6 +459,10 @@ impl Registry { ); self.inject(&[job.as_job_ref()]); job.latch.wait_and_reset(); // Make sure we can use the same latch again next time. + + // flush accumulated logs as we exit the thread + self.logger.log(|| Flush); + job.into_result() }) } @@ -603,6 +616,11 @@ impl WorkerThread { &self.registry } + #[inline] + pub(super) fn log(&self, event: impl FnOnce() -> crate::log::Event) { + self.registry.logger.log(event) + } + /// Our index amongst the worker threads (ranges from `0..self.num_threads()`). #[inline] pub(super) fn index(&self) -> usize { @@ -611,6 +629,7 @@ impl WorkerThread { #[inline] pub(super) unsafe fn push(&self, job: JobRef) { + self.log(|| JobPushed { worker: self.index }); self.worker.push(job); self.registry.sleep.tickle(self.index); } @@ -638,7 +657,6 @@ impl WorkerThread { /// stealing tasks as necessary. #[inline] pub(super) unsafe fn wait_until(&self, latch: &L) { - log!(WaitUntil { worker: self.index }); if !latch.probe() { self.wait_until_cold(latch); } @@ -678,7 +696,7 @@ impl WorkerThread { // wait. self.registry.sleep.work_found(idle_state); - log!(LatchSet { worker: self.index }); + self.log(|| ThreadSawLatchSet { worker: self.index }); mem::forget(abort_guard); // successful execution, do not abort } @@ -716,7 +734,7 @@ impl WorkerThread { match victim.stealer.steal() { Steal::Empty => return None, Steal::Success(d) => { - log!(StoleWork { + self.log(|| JobStolen { worker: self.index, victim: victim_index }); @@ -761,6 +779,9 @@ unsafe fn main_loop(worker: Worker, registry: Arc, index: usiz } } + worker_thread.log(|| ThreadStart { + worker: index, + }); worker_thread.wait_until(®istry.terminate_latch); // Should not be any work left in our queue. @@ -772,6 +793,10 @@ unsafe fn main_loop(worker: Worker, registry: Arc, index: usiz // Normal termination, do not abort. mem::forget(abort_guard); + worker_thread.log(|| ThreadTerminate { + worker: index, + }); + // Inform a user callback that we exited a thread. if let Some(ref handler) = registry.exit_handler { let registry = registry.clone(); diff --git a/rayon-core/src/scope/mod.rs b/rayon-core/src/scope/mod.rs index 7c1333d00..b54d9726b 100644 --- a/rayon-core/src/scope/mod.rs +++ b/rayon-core/src/scope/mod.rs @@ -580,12 +580,12 @@ impl<'scope> ScopeBase<'scope> { .compare_exchange(nil, &mut *err, Ordering::Release, Ordering::Relaxed) .is_ok() { - log!(JobPanickedErrorStored { + self.registry.log(|| JobPanickedErrorStored { owner_thread: self.owner_thread_index }); mem::forget(err); // ownership now transferred into self.panic } else { - log!(JobPanickedErrorNotStored { + self.registry.log(|| JobPanickedErrorNotStored { owner_thread: self.owner_thread_index }); } @@ -594,7 +594,7 @@ impl<'scope> ScopeBase<'scope> { } unsafe fn job_completed_ok(&self) { - log!(JobCompletedOk { + self.registry.log(|| JobCompletedOk { owner_thread: self.owner_thread_index }); self.job_completed_latch.set(); @@ -609,13 +609,13 @@ impl<'scope> ScopeBase<'scope> { // ordering: let panic = self.panic.swap(ptr::null_mut(), Ordering::Relaxed); if !panic.is_null() { - log!(ScopeCompletePanicked { + self.registry.log(|| ScopeCompletePanicked { owner_thread: owner_thread.index() }); let value: Box> = mem::transmute(panic); unwind::resume_unwinding(*value); } else { - log!(ScopeCompleteNoPanic { + self.registry.log(|| ScopeCompleteNoPanic { owner_thread: owner_thread.index() }); } diff --git a/rayon-core/src/sleep/mod.rs b/rayon-core/src/sleep/mod.rs index a464a05d1..ab2e1ce63 100644 --- a/rayon-core/src/sleep/mod.rs +++ b/rayon-core/src/sleep/mod.rs @@ -1,6 +1,7 @@ //! Code that decides when workers should go to sleep. See README.md //! for an overview. +use crate::log::Logger; use crate::log::Event::*; use std::sync::atomic::{AtomicUsize, Ordering}; use std::sync::{Condvar, Mutex}; @@ -11,6 +12,7 @@ pub(super) struct Sleep { state: AtomicUsize, data: Mutex<()>, tickle: Condvar, + logger: Logger, } /// An instance of this struct is created when a thread becomes idle. @@ -33,11 +35,12 @@ const ROUNDS_UNTIL_SLEEPY: usize = 32; const ROUNDS_UNTIL_ASLEEP: usize = 64; impl Sleep { - pub(super) fn new() -> Sleep { + pub(super) fn new(logger: Logger) -> Sleep { Sleep { state: AtomicUsize::new(AWAKE), data: Mutex::new(()), tickle: Condvar::new(), + logger, } } @@ -60,6 +63,10 @@ impl Sleep { #[inline] pub(super) fn start_looking(&self, worker_index: usize) -> IdleState { + self.logger.log(|| ThreadIdle { + worker: worker_index, + }); + IdleState { worker_index, rounds: 0, @@ -70,7 +77,7 @@ impl Sleep { pub(super) fn work_found(&self, idle_state: IdleState) { let IdleState { worker_index, rounds } = idle_state; - log!(FoundWork { + self.logger.log(|| ThreadFoundWork { worker: worker_index, yields: rounds, }); @@ -86,7 +93,7 @@ impl Sleep { #[inline] pub(super) fn no_work_found(&self, idle_state: &mut IdleState) { let IdleState { worker_index, rounds } = idle_state; - log!(DidNotFindWork { + self.logger.log(|| ThreadNoWork { worker: *worker_index, yields: *rounds, }); @@ -105,7 +112,7 @@ impl Sleep { if self.still_sleepy(*worker_index) { *rounds += 1; } else { - log!(GotInterrupted { + self.logger.log(|| ThreadSleepInterrupted { worker: *worker_index }); *rounds = 0; @@ -141,7 +148,7 @@ impl Sleep { // were were going to sleep, we will acquire lock and hence // acquire their reads. let old_state = self.state.swap(AWAKE, Ordering::Release); - log!(Tickle { + self.logger.log(|| Tickle { worker: worker_index, old_state, }); @@ -160,10 +167,6 @@ impl Sleep { // assert a stronger order and acquire any reads etc that // we must see. let state = self.state.load(Ordering::Acquire); - log!(GetSleepy { - worker: worker_index, - state, - }); if self.any_worker_is_sleepy(state) { // somebody else is already sleepy, so we'll just wait our turn debug_assert!( @@ -196,10 +199,10 @@ impl Sleep { .compare_exchange(state, new_state, Ordering::SeqCst, Ordering::Relaxed) .is_ok() { - log!(GotSleepy { + self.logger.log(|| ThreadSleepy { worker: worker_index, - old_state: state, - new_state, + state: state, + new_state: state, }); return true; } @@ -285,17 +288,17 @@ impl Sleep { // to sleep. Note that if we get a false wakeup it's not a // problem for us, we'll just loop around and maybe get // sleepy again. - log!(FellAsleep { + self.logger.log(|| ThreadSleeping { worker: worker_index }); drop(self.tickle.wait(data).unwrap()); - log!(GotAwoken { + self.logger.log(|| ThreadAwoken { worker: worker_index }); return; } } else { - log!(GotInterrupted { + self.logger.log(|| ThreadSleepInterrupted { worker: worker_index }); return; From 2442f67442cd9f565f76763cfa671dac9706f082 Mon Sep 17 00:00:00 2001 From: Niko Matsakis Date: Sun, 3 Nov 2019 06:42:55 -0500 Subject: [PATCH 03/33] remove JobPoppedRhs event and register JobPopped from take_local_job this was needed to register events from `scope` correctly --- rayon-core/src/join/mod.rs | 7 ------- rayon-core/src/log.rs | 8 +------- rayon-core/src/registry.rs | 10 +++++++++- 3 files changed, 10 insertions(+), 15 deletions(-) diff --git a/rayon-core/src/join/mod.rs b/rayon-core/src/join/mod.rs index 37dbf3ea3..aad6e2838 100644 --- a/rayon-core/src/join/mod.rs +++ b/rayon-core/src/join/mod.rs @@ -1,6 +1,5 @@ use crate::job::StackJob; use crate::latch::{LatchProbe, SpinLatch}; -use crate::log::Event::*; use crate::registry::{self, WorkerThread}; use crate::unwind; use std::any::Any; @@ -156,15 +155,9 @@ where // Found it! Let's run it. // // Note that this could panic, but it's ok if we unwind here. - worker_thread.log(|| JobPoppedRhs { - worker: worker_thread.index() - }); let result_b = job_b.run_inline(injected); return (result_a, result_b); } else { - worker_thread.log(|| JobPopped { - worker: worker_thread.index() - }); worker_thread.execute(job); } } else { diff --git a/rayon-core/src/log.rs b/rayon-core/src/log.rs index a62a65172..4b5fb4f2a 100644 --- a/rayon-core/src/log.rs +++ b/rayon-core/src/log.rs @@ -67,12 +67,6 @@ pub(super) enum Event { /// The given worker has popped a job from its local deque. JobPopped { worker: usize }, - /// The given worker has popped a job from its local deque, and - /// the job was the RHS of the `join` we were blocked on. - /// - /// Identical to `JobPopped` but for debugging. - JobPoppedRhs { worker: usize }, - /// The given worker has stolen a job from the deque of another. JobStolen { worker: usize, victim: usize }, @@ -360,7 +354,7 @@ impl SimulatorState { true } - Event::JobPopped { worker } | Event::JobPoppedRhs { worker } => { + Event::JobPopped { worker } => { self.local_queue_size[worker] -= 1; true } diff --git a/rayon-core/src/registry.rs b/rayon-core/src/registry.rs index 9abe27baf..ce85ce50a 100644 --- a/rayon-core/src/registry.rs +++ b/rayon-core/src/registry.rs @@ -650,7 +650,15 @@ impl WorkerThread { /// bottom. #[inline] pub(super) unsafe fn take_local_job(&self) -> Option { - self.worker.pop() + let popped_job = self.worker.pop(); + + if popped_job.is_some() { + self.log(|| JobPopped { + worker: self.index + }); + } + + popped_job } /// Wait until the latch is set. Try to keep busy by popping and From 2458feefed8a2e970ab149d33f3b91c5fe084331 Mon Sep 17 00:00:00 2001 From: Niko Matsakis Date: Sun, 3 Nov 2019 10:36:14 -0500 Subject: [PATCH 04/33] remove pure "debug" events; this lowers overhead of enabling logging --- rayon-core/src/log.rs | 49 +------------------------------------ rayon-core/src/registry.rs | 1 - rayon-core/src/scope/mod.rs | 18 -------------- rayon-core/src/sleep/mod.rs | 21 +--------------- 4 files changed, 2 insertions(+), 87 deletions(-) diff --git a/rayon-core/src/log.rs b/rayon-core/src/log.rs index 4b5fb4f2a..5c075d1f9 100644 --- a/rayon-core/src/log.rs +++ b/rayon-core/src/log.rs @@ -38,23 +38,6 @@ pub(super) enum Event { /// yield rounds. It should no longer be considered idle. ThreadFoundWork { worker: usize, yields: usize }, - /// Indicates that a worker blocked on a latch observed that it was set. - /// - /// Internal debugging event that does not affect the state - /// machine. - ThreadSawLatchSet { worker: usize }, - - /// Indicates that an idle worker thread searched for another round without finding work. - ThreadNoWork { worker: usize, yields: usize }, - - /// Indicates that an idle worker became the sleepy worker. `state` is our internal sleep - /// state value (for debugging). - ThreadSleepy { worker: usize, state: usize, new_state: usize }, - - /// Indicates that the thread's attempt to fall asleep by somebody - /// making us not the sleepy worker. - ThreadSleepInterrupted { worker: usize }, - /// Indicates that an idle worker has gone to sleep. ThreadSleeping { worker: usize }, @@ -75,34 +58,6 @@ pub(super) enum Event { /// A job was removed from the global queue. JobUninjected { worker: usize }, - - /// Indicates that a job completed "ok" as part of a scope. - JobCompletedOk { owner_thread: usize }, - - /// Indicates that a job panicked as part of a scope, and the - /// error was stored for later. - /// - /// Useful for debugging. - JobPanickedErrorStored { owner_thread: usize }, - - /// Indicates that a job panicked as part of a scope, and the - /// error was discarded. - /// - /// Useful for debugging. - JobPanickedErrorNotStored { owner_thread: usize }, - - /// Indicates that a scope completed with a panic. - /// - /// Useful for debugging. - ScopeCompletePanicked { owner_thread: usize }, - - /// Indicates that a scope completed with a panic. - /// - /// Useful for debugging. - ScopeCompleteNoPanic { owner_thread: usize }, - - /// A "tickle" came through and there were sleepy/sleeping workers. - Tickle { worker: usize, old_state: usize }, } /// Handle to the logging thread, if any. You can use this to deliver @@ -374,9 +329,7 @@ impl SimulatorState { true } - // remaining events are no-ops from pov of simulating the - // thread state - _ => false, + Event::Flush => false, } } diff --git a/rayon-core/src/registry.rs b/rayon-core/src/registry.rs index ce85ce50a..8e8c0c840 100644 --- a/rayon-core/src/registry.rs +++ b/rayon-core/src/registry.rs @@ -704,7 +704,6 @@ impl WorkerThread { // wait. self.registry.sleep.work_found(idle_state); - self.log(|| ThreadSawLatchSet { worker: self.index }); mem::forget(abort_guard); // successful execution, do not abort } diff --git a/rayon-core/src/scope/mod.rs b/rayon-core/src/scope/mod.rs index b54d9726b..f2f66bd32 100644 --- a/rayon-core/src/scope/mod.rs +++ b/rayon-core/src/scope/mod.rs @@ -6,7 +6,6 @@ use crate::job::{HeapJob, JobFifo}; use crate::latch::{CountLatch, Latch}; -use crate::log::Event::*; use crate::registry::{in_worker, Registry, WorkerThread}; use crate::unwind; use std::any::Any; @@ -580,23 +579,13 @@ impl<'scope> ScopeBase<'scope> { .compare_exchange(nil, &mut *err, Ordering::Release, Ordering::Relaxed) .is_ok() { - self.registry.log(|| JobPanickedErrorStored { - owner_thread: self.owner_thread_index - }); mem::forget(err); // ownership now transferred into self.panic - } else { - self.registry.log(|| JobPanickedErrorNotStored { - owner_thread: self.owner_thread_index - }); } self.job_completed_latch.set(); } unsafe fn job_completed_ok(&self) { - self.registry.log(|| JobCompletedOk { - owner_thread: self.owner_thread_index - }); self.job_completed_latch.set(); } @@ -609,15 +598,8 @@ impl<'scope> ScopeBase<'scope> { // ordering: let panic = self.panic.swap(ptr::null_mut(), Ordering::Relaxed); if !panic.is_null() { - self.registry.log(|| ScopeCompletePanicked { - owner_thread: owner_thread.index() - }); let value: Box> = mem::transmute(panic); unwind::resume_unwinding(*value); - } else { - self.registry.log(|| ScopeCompleteNoPanic { - owner_thread: owner_thread.index() - }); } } } diff --git a/rayon-core/src/sleep/mod.rs b/rayon-core/src/sleep/mod.rs index ab2e1ce63..e2ca61c9a 100644 --- a/rayon-core/src/sleep/mod.rs +++ b/rayon-core/src/sleep/mod.rs @@ -93,10 +93,6 @@ impl Sleep { #[inline] pub(super) fn no_work_found(&self, idle_state: &mut IdleState) { let IdleState { worker_index, rounds } = idle_state; - self.logger.log(|| ThreadNoWork { - worker: *worker_index, - yields: *rounds, - }); if *rounds < ROUNDS_UNTIL_SLEEPY { thread::yield_now(); *rounds += 1; @@ -112,9 +108,6 @@ impl Sleep { if self.still_sleepy(*worker_index) { *rounds += 1; } else { - self.logger.log(|| ThreadSleepInterrupted { - worker: *worker_index - }); *rounds = 0; } } else { @@ -136,7 +129,7 @@ impl Sleep { } #[cold] - fn tickle_cold(&self, worker_index: usize) { + fn tickle_cold(&self, _worker_index: usize) { // The `Release` ordering here suffices. The reasoning is that // the atomic's own natural ordering ensure that any attempt // to become sleepy/asleep either will come before/after this @@ -148,10 +141,6 @@ impl Sleep { // were were going to sleep, we will acquire lock and hence // acquire their reads. let old_state = self.state.swap(AWAKE, Ordering::Release); - self.logger.log(|| Tickle { - worker: worker_index, - old_state, - }); if self.anyone_sleeping(old_state) { let _data = self.data.lock().unwrap(); self.tickle.notify_all(); @@ -199,11 +188,6 @@ impl Sleep { .compare_exchange(state, new_state, Ordering::SeqCst, Ordering::Relaxed) .is_ok() { - self.logger.log(|| ThreadSleepy { - worker: worker_index, - state: state, - new_state: state, - }); return true; } } @@ -298,9 +282,6 @@ impl Sleep { return; } } else { - self.logger.log(|| ThreadSleepInterrupted { - worker: worker_index - }); return; } } From cddc265373effca04d46356ea2a53e39650f8338 Mon Sep 17 00:00:00 2001 From: Niko Matsakis Date: Sat, 18 Jan 2020 06:41:41 -0500 Subject: [PATCH 05/33] add --skip-bridge option to life The par-bridge test can be really slow and tedious to test. --- rayon-demo/src/life/mod.rs | 30 ++++++++++++++++++------------ 1 file changed, 18 insertions(+), 12 deletions(-) diff --git a/rayon-demo/src/life/mod.rs b/rayon-demo/src/life/mod.rs index b95083883..e2676a1aa 100644 --- a/rayon-demo/src/life/mod.rs +++ b/rayon-demo/src/life/mod.rs @@ -1,6 +1,6 @@ const USAGE: &str = " -Usage: life bench [--size N] [--gens N] - life play [--size N] [--gens N] [--fps N] +Usage: life bench [--size N] [--gens N] [--skip-bridge] + life play [--size N] [--gens N] [--fps N] [--skip-bridge] life --help Conway's Game of Life. @@ -11,6 +11,7 @@ Options: --size N Size of the game board (N x N) [default: 200] --gens N Simulate N generations [default: 100] --fps N Maximum frame rate [default: 60] + --skip-bridge Skips the tests with par-bridge, as it is much slower. -h, --help Show this message. "; @@ -37,6 +38,7 @@ pub struct Args { flag_size: usize, flag_gens: usize, flag_fps: usize, + flag_skip_bridge: bool, } #[derive(PartialEq, Eq, Clone, Debug)] @@ -277,12 +279,14 @@ pub fn main(args: &[String]) { serial as f64 / parallel as f64 ); - let par_bridge = measure(par_bridge_generations, &args).as_nanos(); - println!( - "par_bridge: {:10} ns -> {:.2}x speedup", - par_bridge, - serial as f64 / par_bridge as f64 - ); + if !args.flag_skip_bridge { + let par_bridge = measure(par_bridge_generations, &args).as_nanos(); + println!( + "par_bridge: {:10} ns -> {:.2}x speedup", + par_bridge, + serial as f64 / par_bridge as f64 + ); + } } if args.cmd_play { @@ -298,10 +302,12 @@ pub fn main(args: &[String]) { println!(" cpu usage: {:.1}%", cpu_usage); } - let par_bridge = measure_cpu(par_bridge_generations_limited, &args); - println!("par_bridge: {:.2} fps", par_bridge.actual_fps); - if let Some(cpu_usage) = par_bridge.cpu_usage_percent { - println!(" cpu usage: {:.1}%", cpu_usage); + if !args.flag_skip_bridge { + let par_bridge = measure_cpu(par_bridge_generations_limited, &args); + println!("par_bridge: {:.2} fps", par_bridge.actual_fps); + if let Some(cpu_usage) = par_bridge.cpu_usage_percent { + println!(" cpu usage: {:.1}%", cpu_usage); + } } } } From 2803c5992f59b6606bc436a1f062f19a6a35ee83 Mon Sep 17 00:00:00 2001 From: Niko Matsakis Date: Sat, 18 Jan 2020 13:26:31 -0500 Subject: [PATCH 06/33] add new thread-pool --- rayon-core/src/join/mod.rs | 6 +- rayon-core/src/latch.rs | 228 ++++++++----- rayon-core/src/log.rs | 73 ++++- rayon-core/src/registry.rs | 71 +++-- rayon-core/src/scope/mod.rs | 6 +- rayon-core/src/sleep/counters.rs | 226 +++++++++++++ rayon-core/src/sleep/mod.rs | 530 +++++++++++++++++++------------ 7 files changed, 806 insertions(+), 334 deletions(-) create mode 100644 rayon-core/src/sleep/counters.rs diff --git a/rayon-core/src/join/mod.rs b/rayon-core/src/join/mod.rs index aad6e2838..d72c7e61c 100644 --- a/rayon-core/src/join/mod.rs +++ b/rayon-core/src/join/mod.rs @@ -1,5 +1,5 @@ use crate::job::StackJob; -use crate::latch::{LatchProbe, SpinLatch}; +use crate::latch::SpinLatch; use crate::registry::{self, WorkerThread}; use crate::unwind; use std::any::Any; @@ -133,7 +133,7 @@ where // Create virtual wrapper for task b; this all has to be // done here so that the stack frame can keep it all live // long enough. - let job_b = StackJob::new(call_b(oper_b), SpinLatch::new()); + let job_b = StackJob::new(call_b(oper_b), SpinLatch::new(worker_thread)); let job_b_ref = job_b.as_job_ref(); worker_thread.push(job_b_ref); @@ -179,7 +179,7 @@ where #[cold] // cold path unsafe fn join_recover_from_panic( worker_thread: &WorkerThread, - job_b_latch: &SpinLatch, + job_b_latch: &SpinLatch<'_>, err: Box, ) -> ! { worker_thread.wait_until(job_b_latch); diff --git a/rayon-core/src/latch.rs b/rayon-core/src/latch.rs index fb6e44b00..9d7d9cb93 100644 --- a/rayon-core/src/latch.rs +++ b/rayon-core/src/latch.rs @@ -1,7 +1,8 @@ -use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; -use std::sync::{Arc, Condvar, Mutex}; +use std::sync::atomic::{AtomicUsize, Ordering}; +use std::sync::{Condvar, Mutex}; +use std::usize; -use crate::registry::Registry; +use crate::registry::{Registry, WorkerThread}; /// We define various kinds of latches, which are all a primitive signaling /// mechanism. A latch starts as false. Eventually someone calls `set()` and @@ -29,43 +30,140 @@ use crate::registry::Registry; /// - Once `set()` occurs, the next `probe()` *will* observe it. This /// typically requires a seq-cst ordering. See [the "tickle-then-get-sleepy" scenario in the sleep /// README](/src/sleep/README.md#tickle-then-get-sleepy) for details. -pub(super) trait Latch: LatchProbe { +pub(super) trait Latch { /// Set the latch, signalling others. fn set(&self); } -pub(super) trait LatchProbe { - /// Test if the latch is set. - fn probe(&self) -> bool; +pub(super) trait AsCoreLatch { + fn as_core_latch(&self) -> &CoreLatch; +} + +/// Latch is not set, owning thread is awake +const UNSET: usize = 0; + +/// Latch is not set, owning thread is going to sleep on this latch +/// (but has not yet fallen asleep). +const SLEEPY: usize = 1; + +/// Latch is not set, owning thread is asleep on this latch and +/// must be awoken. +const SLEEPING: usize = 2; + +/// Latch is set. +const SET: usize = 3; + +/// Spin latches are the simplest, most efficient kind, but they do +/// not support a `wait()` operation. They just have a boolean flag +/// that becomes true when `set()` is called. +#[derive(Debug)] +pub(super) struct CoreLatch { + state: AtomicUsize, +} + +impl CoreLatch { + fn new() -> Self { + Self { + state: AtomicUsize::new(0), + } + } + + /// Returns the address of this core latch as an integer. Used + /// for logging. + pub(super) fn addr(&self) -> usize { + self as *const CoreLatch as usize + } + + /// Invoked by owning thread as it prepares to sleep. Returns true + /// if the owning thread may proceed to fall asleep, false if the + /// latch was set in the meantime. + pub(super) fn get_sleepy(&self) -> bool { + self + .state + .compare_exchange(UNSET, SLEEPY, Ordering::SeqCst, Ordering::Relaxed) + .is_ok() + } + + /// Invoked by owning thread as it falls asleep sleep. Returns + /// true if the owning thread should block, or false if the latch + /// was set in the meantime. + pub(super) fn fall_asleep(&self) -> bool { + self + .state + .compare_exchange(SLEEPY, SLEEPING, Ordering::SeqCst, Ordering::Relaxed) + .is_ok() + } + + /// Invoked by owning thread as it falls asleep sleep. Returns + /// true if the owning thread should block, or false if the latch + /// was set in the meantime. + pub(super) fn wake_up(&self) { + if !self.probe() { + let _ = + self + .state + .compare_exchange(SLEEPING, UNSET, Ordering::SeqCst, Ordering::Relaxed); + } + } + + /// Set the latch. If this returns true, the owning thread was sleeping + /// and must be awoken. + /// + /// This is private because, typically, setting a latch involves + /// doing some wakeups; those are encapsulated in the surrounding + /// latch code. + fn set(&self) -> bool { + let old_state = self.state.swap(SET, Ordering::AcqRel); + old_state == SLEEPING + } + + /// Test if this latch has been set. + pub(super) fn probe(&self) -> bool { + self.state.load(Ordering::Acquire) == SET + } } /// Spin latches are the simplest, most efficient kind, but they do /// not support a `wait()` operation. They just have a boolean flag /// that becomes true when `set()` is called. -pub(super) struct SpinLatch { - b: AtomicBool, +pub(super) struct SpinLatch<'r> { + core_latch: CoreLatch, + registry: &'r Registry, + target_worker_index: usize, } -impl SpinLatch { +impl<'r> SpinLatch<'r> { + /// Creates a new spin latch that is owned by `thread`. This means + /// that `thread` is the only thread that should be blocking on + /// this latch -- it also means that when the latch is set, we + /// will wake `thread` if it is sleeping. #[inline] - pub(super) fn new() -> SpinLatch { + pub(super) fn new(thread: &'r WorkerThread) -> SpinLatch<'r> { SpinLatch { - b: AtomicBool::new(false), + core_latch: CoreLatch::new(), + registry: thread.registry(), + target_worker_index: thread.index(), } } + + pub(super) fn probe(&self) -> bool { + self.core_latch.probe() + } } -impl LatchProbe for SpinLatch { +impl<'r> AsCoreLatch for SpinLatch<'r> { #[inline] - fn probe(&self) -> bool { - self.b.load(Ordering::SeqCst) + fn as_core_latch(&self) -> &CoreLatch { + &self.core_latch } } -impl Latch for SpinLatch { +impl<'r> Latch for SpinLatch<'r> { #[inline] fn set(&self) { - self.b.store(true, Ordering::SeqCst); + if self.core_latch.set() { + self.registry.notify_worker_latch_is_set(self.target_worker_index); + } } } @@ -103,15 +201,6 @@ impl LockLatch { } } -impl LatchProbe for LockLatch { - #[inline] - fn probe(&self) -> bool { - // Not particularly efficient, but we don't really use this operation - let guard = self.m.lock().unwrap(); - *guard - } -} - impl Latch for LockLatch { #[inline] fn set(&self) { @@ -126,8 +215,19 @@ impl Latch for LockLatch { /// necessarily make the latch be considered `set()`; instead, it just /// decrements the counter. The latch is only "set" (in the sense that /// `probe()` returns true) once the counter reaches zero. +/// +/// Note: like a `SpinLatch`, count laches are always associated with +/// some registry that is probing them, which must be tickled when +/// they are set. *Unlike* a `SpinLatch`, they don't themselves hold a +/// reference to that registry. This is because in some cases the +/// registry owns the count-latch, and that would create a cycle. So a +/// `CountLatch` must be given a reference to its owning registry when +/// it is set. For this reason, it does not implement the `Latch` +/// trait (but it doesn't have to, as it is not used in those generic +/// contexts). #[derive(Debug)] pub(super) struct CountLatch { + core_latch: CoreLatch, counter: AtomicUsize, } @@ -135,77 +235,47 @@ impl CountLatch { #[inline] pub(super) fn new() -> CountLatch { CountLatch { + core_latch: CoreLatch::new(), counter: AtomicUsize::new(1), } } #[inline] pub(super) fn increment(&self) { - debug_assert!(!self.probe()); + debug_assert!(!self.core_latch.probe()); self.counter.fetch_add(1, Ordering::Relaxed); } -} - -impl LatchProbe for CountLatch { - #[inline] - fn probe(&self) -> bool { - // Need to acquire any memory reads before latch was set: - self.counter.load(Ordering::SeqCst) == 0 - } -} - -impl Latch for CountLatch { - /// Set the latch to true, releasing all threads who are waiting. - #[inline] - fn set(&self) { - self.counter.fetch_sub(1, Ordering::SeqCst); - } -} -/// A tickling latch wraps another latch type, and will also awaken a thread -/// pool when it is set. This is useful for jobs injected between thread pools, -/// so the source pool can continue processing its own work while waiting. -pub(super) struct TickleLatch<'a, L: Latch> { - inner: L, - registry: &'a Arc, -} - -impl<'a, L: Latch> TickleLatch<'a, L> { + /// Decrements the latch counter by one. If this is the final + /// count, then the latch is **set**, and calls to `probe()` will + /// return true. Returns whether the latch was set. This is an + /// internal operation, as it does not tickle, and to fail to + /// tickle would lead to deadlock. #[inline] - pub(super) fn new(latch: L, registry: &'a Arc) -> Self { - registry.increment_terminate_count(); - TickleLatch { - inner: latch, - registry, + fn set(&self) -> bool { + if self.counter.fetch_sub(1, Ordering::SeqCst) == 1 { + self.core_latch.set(); + true + } else { + false } } -} -impl<'a, L: Latch> LatchProbe for TickleLatch<'a, L> { + /// Decrements the latch counter by one and possibly set it. If + /// the latch is set, then the specific worker thread is tickled, + /// which should be the one that owns this latch. #[inline] - fn probe(&self) -> bool { - self.inner.probe() + pub(super) fn set_and_tickle_one(&self, registry: &Registry, target_worker_index: usize) { + if self.set() { + registry.notify_worker_latch_is_set(target_worker_index); + } } } -impl<'a, L: Latch> Latch for TickleLatch<'a, L> { +impl AsCoreLatch for CountLatch { #[inline] - fn set(&self) { - // Ensure the registry stays alive while we tickle it. - let registry = Arc::clone(self.registry); - - // NOTE: Once we `set`, the target may proceed and invalidate `&self`! - self.inner.set(); - registry.tickle(); - } -} - -impl<'a, L> LatchProbe for &'a L -where - L: LatchProbe, -{ - fn probe(&self) -> bool { - L::probe(self) + fn as_core_latch(&self) -> &CoreLatch { + &self.core_latch } } diff --git a/rayon-core/src/log.rs b/rayon-core/src/log.rs index 5c075d1f9..34eb9b9f9 100644 --- a/rayon-core/src/log.rs +++ b/rayon-core/src/log.rs @@ -24,6 +24,7 @@ pub(super) enum Event { /// Indicates that a worker thread started execution. ThreadStart { worker: usize, + terminate_addr: usize, }, /// Indicates that a worker thread started execution. @@ -31,18 +32,42 @@ pub(super) enum Event { worker: usize, }, - /// Indicates that a worker thread became idle, waiting for a latch. - ThreadIdle { worker: usize }, + /// Indicates that a worker thread became idle, blocked on `latch_addr`. + ThreadIdle { worker: usize, latch_addr: usize }, /// Indicates that an idle worker thread found work to do, after /// yield rounds. It should no longer be considered idle. - ThreadFoundWork { worker: usize, yields: usize }, + ThreadFoundWork { worker: usize, yields: u32 }, + + /// Indicates that a worker blocked on a latch observed that it was set. + /// + /// Internal debugging event that does not affect the state + /// machine. + ThreadSawLatchSet { worker: usize, latch_addr: usize }, + + /// Indicates that an idle worker is getting sleepy. `sleepy_counter` is the internal + /// sleep state that we saw at the time. + ThreadSleepy { worker: usize, sleepy_counter: u16 }, + + /// Indicates that the thread's attempt to fall asleep was + /// interrupted because the latch was set. (This is not, in and of + /// itself, a change to the thread state.) + ThreadSleepInterruptedByLatch { worker: usize, latch_addr: usize }, + + /// Indicates that the thread's attempt to fall asleep was + /// interrupted because a job was posted. (This is not, in and of + /// itself, a change to the thread state.) + ThreadSleepInterruptedByJob { worker: usize }, /// Indicates that an idle worker has gone to sleep. - ThreadSleeping { worker: usize }, + ThreadSleeping { worker: usize, latch_addr: usize }, /// Indicates that a sleeping worker has awoken. - ThreadAwoken { worker: usize }, + ThreadAwoken { worker: usize, latch_addr: usize }, + + /// Indicates that the given worker thread was notified it should + /// awaken. + ThreadNotify { worker: usize }, /// The given worker has pushed a job to its local deque. JobPushed { worker: usize }, @@ -58,6 +83,15 @@ pub(super) enum Event { /// A job was removed from the global queue. JobUninjected { worker: usize }, + + /// When announcing a job, this was the value of the counters we observed. + /// + /// No effect on thread state, just a debugging event. + JobThreadCounts { + worker: usize, + num_idle: u16, + num_sleepers: u16, + }, } /// Handle to the logging thread, if any. You can use this to deliver @@ -100,7 +134,7 @@ impl Logger { Self::profile_logger_thread(num_workers, filename, 10_000, receiver) }); } else { - panic!("RAYON_RS_LOG should be 'all', 'tail:', or 'profile:'"); + panic!("RAYON_RS_LOG should be 'tail:' or 'profile:'"); } return Logger { @@ -143,7 +177,7 @@ impl Logger { loop { match receiver.recv_timeout(timeout) { Ok(event) => { - if let Event::Flush = event { + if let Event::Flush = event { break; } else { events.push(event); @@ -243,6 +277,7 @@ impl Logger { enum State { Working, Idle, + Notified, Sleeping, Terminated, } @@ -252,7 +287,7 @@ impl State { match self { State::Working => 'W', State::Idle => 'I', - + State::Notified => 'N', State::Sleeping => 'S', State::Terminated => 'T', } @@ -299,7 +334,7 @@ impl SimulatorState { } Event::ThreadAwoken { worker, .. } => { - assert_eq!(self.thread_states[worker], State::Sleeping); + assert_eq!(self.thread_states[worker], State::Notified); self.thread_states[worker] = State::Idle; true } @@ -329,7 +364,18 @@ impl SimulatorState { true } - Event::Flush => false, + Event::ThreadNotify { worker } => { + // Currently, this log event occurs while holding the + // thread lock, so we should *always* see it before + // the worker awakens. + assert_eq!(self.thread_states[worker], State::Sleeping); + self.thread_states[worker] = State::Notified; + true + } + + // remaining events are no-ops from pov of simulating the + // thread state + _ => false, } } @@ -346,8 +392,11 @@ impl SimulatorState { .filter(|s| **s == State::Sleeping) .count(); - // we don't track this on rayon master, but other branches do - let num_notified_threads = 0; + let num_notified_threads = self + .thread_states + .iter() + .filter(|s| **s == State::Notified) + .count(); let num_pending_jobs: usize = self.local_queue_size.iter().sum(); diff --git a/rayon-core/src/registry.rs b/rayon-core/src/registry.rs index 8e8c0c840..037013c72 100644 --- a/rayon-core/src/registry.rs +++ b/rayon-core/src/registry.rs @@ -1,5 +1,5 @@ use crate::job::{JobFifo, JobRef, StackJob}; -use crate::latch::{CountLatch, Latch, LatchProbe, LockLatch, SpinLatch, TickleLatch}; +use crate::latch::{AsCoreLatch, CoreLatch, CountLatch, Latch, LockLatch, SpinLatch}; use crate::log::Logger; use crate::log::Event::*; use crate::sleep::Sleep; @@ -154,7 +154,7 @@ pub(super) struct Registry { // - when `join()` or `scope()` is invoked, similarly, no adjustments are needed. // These are always owned by some other job (e.g., one injected by `ThreadPool::install()`) // and that job will keep the pool alive. - terminate_latch: CountLatch, + terminate_count: AtomicUsize, } /// //////////////////////////////////////////////////////////////////////// @@ -239,9 +239,9 @@ impl Registry { let registry = Arc::new(Registry { logger: logger.clone(), thread_infos: stealers.into_iter().map(ThreadInfo::new).collect(), - sleep: Sleep::new(logger), + sleep: Sleep::new(logger, n_threads), injected_jobs: SegQueue::new(), - terminate_latch: CountLatch::new(), + terminate_count: AtomicUsize::new(1), panic_handler: builder.take_panic_handler(), start_handler: builder.take_start_handler(), exit_handler: builder.take_exit_handler(), @@ -392,15 +392,19 @@ impl Registry { // drops) a `ThreadPool`; and, in that case, they cannot be // calling `inject()` later, since they dropped their // `ThreadPool`. - assert!( - !self.terminate_latch.probe(), + debug_assert_ne!( + self.terminate_count.load(Ordering::Acquire), + 0, "inject() sees state.terminate as true" ); + let queue_was_empty = self.injected_jobs.is_empty(); + for &job_ref in injected_jobs { self.injected_jobs.push(job_ref); } - self.sleep.tickle(usize::MAX); + + self.sleep.new_injected_jobs(usize::MAX, injected_jobs.len() as u32, queue_was_empty); } fn pop_injected_job(&self, worker_index: usize) -> Option { @@ -476,7 +480,7 @@ impl Registry { // This thread is a member of a different pool, so let it process // other work while waiting for this `op` to complete. debug_assert!(current_thread.registry().id() != self.id()); - let latch = TickleLatch::new(SpinLatch::new(), current_thread.registry()); + let latch = SpinLatch::new(current_thread); let job = StackJob::new( |injected| { let worker_thread = WorkerThread::current(); @@ -511,19 +515,25 @@ impl Registry { /// terminate count and is responsible for invoking `terminate()` /// when finished. pub(super) fn increment_terminate_count(&self) { - self.terminate_latch.increment(); + let previous = self.terminate_count.fetch_add(1, Ordering::AcqRel); + debug_assert!(previous != 0, "registry ref count incremented from zero"); + assert!(previous != std::usize::MAX, "overflow in registry ref count"); } /// Signals that the thread-pool which owns this registry has been /// dropped. The worker threads will gradually terminate, once any /// extant work is completed. pub(super) fn terminate(&self) { - self.terminate_latch.set(); - self.sleep.tickle(usize::MAX); + if self.terminate_count.fetch_sub(1, Ordering::AcqRel) == 1 { + for (i, thread_info) in self.thread_infos.iter().enumerate() { + thread_info.terminate.set_and_tickle_one(self, i); + } + } } - pub(super) fn tickle(&self) { - self.sleep.tickle(usize::MAX); + /// Notify the worker that the latch they are sleeping on has bene "set". + pub(super) fn notify_worker_latch_is_set(&self, target_worker_index: usize) { + self.sleep.notify_worker_latch_is_set(target_worker_index); } } @@ -542,6 +552,15 @@ struct ThreadInfo { /// until workers have stopped; only used for tests. stopped: LockLatch, + /// The latch used to signal that terminated has been requested. + /// This latch is *set* by the `terminate` method on the + /// `Registry`, once the registry's main "terminate" counter + /// reaches zero. + /// + /// NB. We use a `CountLatch` here because it has no lifetimes and is + /// meant for async use, but the count never gets higher than one. + terminate: CountLatch, + /// the "stealer" half of the worker's deque stealer: Stealer, } @@ -551,6 +570,7 @@ impl ThreadInfo { ThreadInfo { primed: LockLatch::new(), stopped: LockLatch::new(), + terminate: CountLatch::new(), stealer, } } @@ -630,8 +650,9 @@ impl WorkerThread { #[inline] pub(super) unsafe fn push(&self, job: JobRef) { self.log(|| JobPushed { worker: self.index }); + let queue_was_empty = self.worker.is_empty(); self.worker.push(job); - self.registry.sleep.tickle(self.index); + self.registry.sleep.new_internal_jobs(self.index, 1, queue_was_empty); } #[inline] @@ -664,14 +685,15 @@ impl WorkerThread { /// Wait until the latch is set. Try to keep busy by popping and /// stealing tasks as necessary. #[inline] - pub(super) unsafe fn wait_until(&self, latch: &L) { + pub(super) unsafe fn wait_until(&self, latch: &L) { + let latch = latch.as_core_latch(); if !latch.probe() { self.wait_until_cold(latch); } } #[cold] - unsafe fn wait_until_cold(&self, latch: &L) { + unsafe fn wait_until_cold(&self, latch: &CoreLatch) { // the code below should swallow all panics and hence never // unwind; but if something does wrong, we want to abort, // because otherwise other code in rayon may assume that the @@ -679,7 +701,7 @@ impl WorkerThread { // accesses, which would be *very bad* let abort_guard = unwind::AbortIfPanic; - let mut idle_state = self.registry.sleep.start_looking(self.index); + let mut idle_state = self.registry.sleep.start_looking(self.index, latch); while !latch.probe() { // Try to find some work to do. We give preference first // to things in our local deque, then in other workers @@ -693,9 +715,9 @@ impl WorkerThread { { self.registry.sleep.work_found(idle_state); self.execute(job); - idle_state = self.registry.sleep.start_looking(self.index); + idle_state = self.registry.sleep.start_looking(self.index, latch); } else { - self.registry.sleep.no_work_found(&mut idle_state); + self.registry.sleep.no_work_found(&mut idle_state, latch); } } @@ -704,17 +726,12 @@ impl WorkerThread { // wait. self.registry.sleep.work_found(idle_state); + self.log(|| ThreadSawLatchSet { worker: self.index, latch_addr: latch.addr() }); mem::forget(abort_guard); // successful execution, do not abort } pub(super) unsafe fn execute(&self, job: JobRef) { job.execute(); - - // Subtle: executing this job will have `set()` some of its - // latches. This may mean that a sleepy (or sleeping) worker - // can now make progress. So we have to tickle them to let - // them know. - self.registry.sleep.tickle(self.index); } /// Try to steal a single job and return it. @@ -786,10 +803,12 @@ unsafe fn main_loop(worker: Worker, registry: Arc, index: usiz } } + let my_terminate_latch = ®istry.thread_infos[index].terminate; worker_thread.log(|| ThreadStart { worker: index, + terminate_addr: my_terminate_latch.as_core_latch().addr(), }); - worker_thread.wait_until(®istry.terminate_latch); + worker_thread.wait_until(my_terminate_latch); // Should not be any work left in our queue. debug_assert!(worker_thread.take_local_job().is_none()); diff --git a/rayon-core/src/scope/mod.rs b/rayon-core/src/scope/mod.rs index f2f66bd32..296a37f35 100644 --- a/rayon-core/src/scope/mod.rs +++ b/rayon-core/src/scope/mod.rs @@ -5,7 +5,7 @@ //! [`join()`]: ../join/join.fn.html use crate::job::{HeapJob, JobFifo}; -use crate::latch::{CountLatch, Latch}; +use crate::latch::CountLatch; use crate::registry::{in_worker, Registry, WorkerThread}; use crate::unwind; use std::any::Any; @@ -582,11 +582,11 @@ impl<'scope> ScopeBase<'scope> { mem::forget(err); // ownership now transferred into self.panic } - self.job_completed_latch.set(); + self.job_completed_latch.set_and_tickle_one(&self.registry, self.owner_thread_index); } unsafe fn job_completed_ok(&self) { - self.job_completed_latch.set(); + self.job_completed_latch.set_and_tickle_one(&self.registry, self.owner_thread_index); } unsafe fn steal_till_jobs_complete(&self, owner_thread: &WorkerThread) { diff --git a/rayon-core/src/sleep/counters.rs b/rayon-core/src/sleep/counters.rs new file mode 100644 index 000000000..5fc4e5cf4 --- /dev/null +++ b/rayon-core/src/sleep/counters.rs @@ -0,0 +1,226 @@ +use std::sync::atomic::{AtomicU64, Ordering}; + +pub(super) struct AtomicCounters { + /// Packs together a number of counters. + /// + /// Consider the value `0x1111_2222_3333_4444`: + /// + /// - The bits 0x1111 are the **jobs counter**. + /// - The bits 0x2222 are the **sleepy counter**. + /// - The bits 0x3333 are the number of **idle threads**. + /// - The bits 0x4444 are the number of **sleeping threads**. + /// + /// See the struct `Counters` below. + value: AtomicU64 +} + +#[derive(Copy, Clone)] +pub(super) struct Counters { + word: u64 +} + +#[derive(Copy, Clone, Debug, PartialEq, PartialOrd)] +pub(super) struct SleepyCounter(u16); + +#[derive(Copy, Clone, Debug, PartialEq, PartialOrd)] +pub(super) struct JobsCounter(u16); + +const ONE_SLEEPING: u64 = 0x0000_0000_0000_0001; +const ONE_IDLE: u64 = 0x0000_0000_0001_0000; +const ONE_SLEEPY: u64 = 0x0000_0001_0000_0000; +const SLEEPY_ROLLVER_MASK: u64 = 0x0000_0000_FFFF_FFFF; +const NO_JOBS_MASK: u64 = 0x0000_FFFF_FFFF_FFFF; +const SLEEPING_SHIFT: u64 = 0; +const IDLE_SHIFT: u64 = 1 * 16; +const SLEEPY_SHIFT: u64 = 2 * 16; +const JOBS_SHIFT: u64 = 3 * 16; + +pub(super) const INVALID_SLEEPY_COUNTER: SleepyCounter = SleepyCounter(std::u16::MAX); +pub(super) const ZERO_SLEEPY_COUNTER: SleepyCounter = SleepyCounter(0); + +impl AtomicCounters { + pub(super) fn new() -> AtomicCounters { + AtomicCounters { value: AtomicU64::new(0) } + } + + pub(super) fn load(&self, ordering: Ordering) -> Counters { + Counters::new(self.value.load(ordering)) + } + + #[inline] + fn try_exchange(&self, old_value: Counters, new_value: Counters, ordering: Ordering) -> bool { + self.value.compare_exchange( + old_value.word, + new_value.word, + ordering, + Ordering::Relaxed, + ).is_ok() + } + + /// Adds an idle thread. This cannot fail. + #[inline] + pub(super) fn add_idle_thread(&self) { + self.value.fetch_add(ONE_IDLE, Ordering::SeqCst); + } + + #[inline] + pub(super) fn try_add_sleepy_thread(&self, old_value: Counters) -> bool { + debug_assert!( + !old_value.sleepy_counter().is_max(), + "try_add_sleepy_thread: old_value {:?} has max sleepy threads", + old_value, + ); + let new_value = Counters::new(old_value.word + ONE_SLEEPY); + self.try_exchange(old_value, new_value, Ordering::SeqCst) + } + + /// Subtracts an idle thread. This cannot fail; it returns the + /// number of sleeping threads to wake up (if any). + #[inline] + pub(super) fn sub_idle_thread(&self) -> u16 { + let old_value = Counters::new(self.value.fetch_sub(ONE_IDLE, Ordering::SeqCst)); + debug_assert!( + old_value.raw_idle_threads() > 0, + "sub_idle_thread: old_value {:?} has no idle threads", + old_value, + ); + + // Current heuristic: whenever an idle thread goes away, if + // there are any sleeping threads, wake 'em up. + let sleeping_threads = old_value.sleeping_threads(); + std::cmp::min(sleeping_threads, 2) + } + + /// Subtracts a sleeping thread. This cannot fail, but it is only + /// safe to do if you you know the number of sleeping threads is + /// non-zero (i.e., because you have just awoken a sleeping + /// thread). + #[inline] + pub(super) fn sub_sleeping_thread(&self) { + let old_value = Counters::new(self.value.fetch_sub(ONE_SLEEPING, Ordering::SeqCst)); + debug_assert!( + old_value.sleeping_threads() > 0, + "sub_sleeping_thread: old_value {:?} had no sleeping threads", + old_value, + ); + } + + #[inline] + pub(super) fn try_add_sleeping_thread(&self, old_value: Counters) -> bool { + debug_assert!( + old_value.raw_idle_threads() > 0, + "try_add_sleeping_thread: old_value {:?} has no idle threads", + old_value, + ); + debug_assert!( + old_value.sleeping_threads() < std::u16::MAX, + "try_add_sleeping_thread: old_value {:?} has too many sleeping threads", + old_value, + ); + + let mut new_value = old_value; + new_value.word += ONE_SLEEPING; + + self.try_exchange(old_value, new_value, Ordering::SeqCst) + } + + #[inline] + pub(super) fn try_replicate_sleepy_counter(&self, old_value: Counters) -> bool { + let sc = old_value.sleepy_counter(); + + // clear jobs counter + let word_without_jc = old_value.word & NO_JOBS_MASK; + + // replace with sleepy counter + let sc_shifted_to_jc = (sc.0 as u64) << JOBS_SHIFT; + let new_value = Counters::new(word_without_jc | sc_shifted_to_jc); + + debug_assert!(new_value.sleepy_counter() == old_value.sleepy_counter()); + debug_assert!(new_value.jobs_counter() == old_value.sleepy_counter()); + + self.try_exchange(old_value, new_value, Ordering::SeqCst) + } + + #[inline] + pub(super) fn try_rollover_jobs_and_sleepy_counters(&self, old_value: Counters) -> bool { + let new_value = Counters::new(old_value.word & SLEEPY_ROLLVER_MASK); + self.try_exchange(old_value, new_value, Ordering::SeqCst) + } +} + +fn select_u16(word: u64, shift: u64) -> u16 { + (word >> shift) as u16 +} + +impl Counters { + fn new(word: u64) -> Counters { + Counters { word } + } + + pub(super) fn jobs_counter(self) -> JobsCounter { + JobsCounter(select_u16(self.word, JOBS_SHIFT)) + } + + pub(super) fn sleepy_counter(self) -> SleepyCounter { + SleepyCounter(select_u16(self.word, SLEEPY_SHIFT)) + } + + pub(super) fn raw_idle_threads(self) -> u16 { + select_u16(self.word, IDLE_SHIFT) + } + + pub(super) fn awake_but_idle_threads(self) -> u16 { + self.raw_idle_threads() - self.sleeping_threads() + } + + pub(super) fn sleeping_threads(self) -> u16 { + select_u16(self.word, SLEEPING_SHIFT) + } +} + +impl SleepyCounter { + pub(super) fn is_max(self) -> bool { + self.0 == std::u16::MAX + } + + pub(super) fn as_u16(self) -> u16 { + self.0 + } +} + +impl PartialOrd for SleepyCounter { + fn partial_cmp(&self, other: &JobsCounter) -> Option<::std::cmp::Ordering> { + PartialOrd::partial_cmp(&self.0, &other.0) + } +} + +impl PartialOrd for JobsCounter { + fn partial_cmp(&self, other: &SleepyCounter) -> Option<::std::cmp::Ordering> { + PartialOrd::partial_cmp(&self.0, &other.0) + } +} + +impl PartialEq for SleepyCounter { + fn eq(&self, other: &JobsCounter) -> bool { + PartialEq::eq(&self.0, &other.0) + } +} + +impl PartialEq for JobsCounter { + fn eq(&self, other: &SleepyCounter) -> bool { + PartialEq::eq(&self.0, &other.0) + } +} + +impl std::fmt::Debug for Counters { + fn fmt(&self, fmt: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let word = format!("{:016x}", self.word); + fmt.debug_struct("Counters") + .field("word", &word) + .field("jobs", &self.jobs_counter().0) + .field("sleepy", &self.sleepy_counter()) + .field("idle", &self.raw_idle_threads()) + .field("sleeping", &self.sleeping_threads()) + .finish() + } +} diff --git a/rayon-core/src/sleep/mod.rs b/rayon-core/src/sleep/mod.rs index e2ca61c9a..87b498531 100644 --- a/rayon-core/src/sleep/mod.rs +++ b/rayon-core/src/sleep/mod.rs @@ -1,18 +1,26 @@ //! Code that decides when workers should go to sleep. See README.md //! for an overview. +use crossbeam_utils::CachePadded; +use crate::latch::CoreLatch; use crate::log::Logger; use crate::log::Event::*; -use std::sync::atomic::{AtomicUsize, Ordering}; +use std::sync::atomic::Ordering; use std::sync::{Condvar, Mutex}; use std::thread; use std::usize; +mod counters; +use self::counters::{AtomicCounters, Counters, INVALID_SLEEPY_COUNTER, SleepyCounter, ZERO_SLEEPY_COUNTER}; + pub(super) struct Sleep { - state: AtomicUsize, - data: Mutex<()>, - tickle: Condvar, logger: Logger, + + /// One "sleep state" per worker. Used to track if a worker is sleeping and to have + /// them block. + worker_sleep_states: Vec>, + + counters: AtomicCounters, } /// An instance of this struct is created when a thread becomes idle. @@ -25,265 +33,365 @@ pub(super) struct IdleState { worker_index: usize, /// How many rounds have we been circling without sleeping? - rounds: usize, + rounds: u32, + + /// Once we become sleepy, what was the sleepy counter value? + /// Set to `INVALID_SLEEPY_COUNTER` otherwise. + sleepy_counter: SleepyCounter, } -const AWAKE: usize = 0; -const SLEEPING: usize = 1; +/// The "sleep state" for an individual worker. +#[derive(Default)] +struct WorkerSleepState { + /// Set to true when the worker goes to sleep; set to false when + /// the worker is notified or when it wakes. + is_blocked: Mutex, -const ROUNDS_UNTIL_SLEEPY: usize = 32; -const ROUNDS_UNTIL_ASLEEP: usize = 64; + condvar: Condvar, +} + +const ROUNDS_UNTIL_SLEEPY: u32 = 32; +const ROUNDS_UNTIL_SLEEPING: u32 = ROUNDS_UNTIL_SLEEPY + 1; impl Sleep { - pub(super) fn new(logger: Logger) -> Sleep { + pub(super) fn new(logger: Logger, n_threads: usize) -> Sleep { Sleep { - state: AtomicUsize::new(AWAKE), - data: Mutex::new(()), - tickle: Condvar::new(), logger, + worker_sleep_states: (0..n_threads).map(|_| Default::default()).collect(), + counters: AtomicCounters::new(), } } - fn anyone_sleeping(&self, state: usize) -> bool { - state & SLEEPING != 0 - } - - fn any_worker_is_sleepy(&self, state: usize) -> bool { - (state >> 1) != 0 - } - - fn worker_is_sleepy(&self, state: usize, worker_index: usize) -> bool { - (state >> 1) == (worker_index + 1) - } - - fn with_sleepy_worker(&self, state: usize, worker_index: usize) -> usize { - debug_assert!(state == AWAKE || state == SLEEPING); - ((worker_index + 1) << 1) + state - } - #[inline] - pub(super) fn start_looking(&self, worker_index: usize) -> IdleState { + pub(super) fn start_looking(&self, worker_index: usize, latch: &CoreLatch) -> IdleState { self.logger.log(|| ThreadIdle { worker: worker_index, + latch_addr: latch.addr(), }); + self.counters.add_idle_thread(); + IdleState { worker_index, rounds: 0, + sleepy_counter: INVALID_SLEEPY_COUNTER, } } #[inline] pub(super) fn work_found(&self, idle_state: IdleState) { - let IdleState { worker_index, rounds } = idle_state; - self.logger.log(|| ThreadFoundWork { - worker: worker_index, - yields: rounds, + worker: idle_state.worker_index, + yields: idle_state.rounds, }); - if rounds > ROUNDS_UNTIL_SLEEPY { - // FIXME tickling here is a bit extreme; mostly we want to "release the lock" - // from us being sleepy, we don't necessarily need to wake others - // who are sleeping - self.tickle(worker_index); - } + // If we were the last idle thread and other threads are still sleeping, + // then we should wake up another thread. + let threads_to_wake = self.counters.sub_idle_thread(); + self.wake_any_threads(threads_to_wake as u32); } #[inline] - pub(super) fn no_work_found(&self, idle_state: &mut IdleState) { - let IdleState { worker_index, rounds } = idle_state; - if *rounds < ROUNDS_UNTIL_SLEEPY { + pub(super) fn no_work_found( + &self, + idle_state: &mut IdleState, + latch: &CoreLatch, + ) { + if idle_state.rounds < ROUNDS_UNTIL_SLEEPY { thread::yield_now(); - *rounds += 1; - } else if *rounds == ROUNDS_UNTIL_SLEEPY { + idle_state.rounds += 1; + } else if idle_state.rounds == ROUNDS_UNTIL_SLEEPY { + idle_state.sleepy_counter = self.announce_sleepy(idle_state.worker_index); + idle_state.rounds += 1; thread::yield_now(); - if self.get_sleepy(*worker_index) { - *rounds += 1; - } else { - *rounds = 0; - } - } else if *rounds < ROUNDS_UNTIL_ASLEEP { + } else if idle_state.rounds < ROUNDS_UNTIL_SLEEPING { + idle_state.rounds += 1; thread::yield_now(); - if self.still_sleepy(*worker_index) { - *rounds += 1; - } else { - *rounds = 0; - } } else { - debug_assert_eq!(*rounds, ROUNDS_UNTIL_ASLEEP); - self.sleep(*worker_index); - *rounds = 0; + debug_assert_eq!(idle_state.rounds, ROUNDS_UNTIL_SLEEPING); + self.sleep(idle_state, latch); } } - pub(super) fn tickle(&self, worker_index: usize) { - // As described in README.md, this load must be SeqCst so as to ensure that: - // - if anyone is sleepy or asleep, we *definitely* see that now (and not eventually); - // - if anyone after us becomes sleepy or asleep, they see memory events that - // precede the call to `tickle()`, even though we did not do a write. - let old_state = self.state.load(Ordering::SeqCst); - if old_state != AWAKE { - self.tickle_cold(worker_index); + #[cold] + fn announce_sleepy(&self, worker_index: usize) -> SleepyCounter { + loop { + let counters = self.counters.load(Ordering::Relaxed); + let sleepy_counter = counters.sleepy_counter(); + if sleepy_counter.is_max() { + if self.counters.try_rollover_jobs_and_sleepy_counters(counters) { + self.logger.log(|| ThreadSleepy { worker: worker_index, sleepy_counter: 0 }); + return ZERO_SLEEPY_COUNTER; + } + } else { + if self.counters.try_add_sleepy_thread(counters) { + self.logger.log(|| ThreadSleepy { worker: worker_index, sleepy_counter: sleepy_counter.as_u16() }); + return sleepy_counter; + } + } } } #[cold] - fn tickle_cold(&self, _worker_index: usize) { - // The `Release` ordering here suffices. The reasoning is that - // the atomic's own natural ordering ensure that any attempt - // to become sleepy/asleep either will come before/after this - // swap. If it comes *after*, then Release is good because we - // want it to see the action that generated this tickle. If it - // comes *before*, then we will see it here (but not other - // memory writes from that thread). If the other worker was - // becoming sleepy, the other writes don't matter. If they - // were were going to sleep, we will acquire lock and hence - // acquire their reads. - let old_state = self.state.swap(AWAKE, Ordering::Release); - if self.anyone_sleeping(old_state) { - let _data = self.data.lock().unwrap(); - self.tickle.notify_all(); + fn sleep(&self, idle_state: &mut IdleState, latch: &CoreLatch) { + let worker_index = idle_state.worker_index; + + if !latch.get_sleepy() { + self.logger.log(|| ThreadSleepInterruptedByLatch { + worker: worker_index, + latch_addr: latch.addr(), + }); + + return; + } + + let sleep_state = &self.worker_sleep_states[worker_index]; + let mut is_blocked = sleep_state.is_blocked.lock().unwrap(); + debug_assert!(!*is_blocked); + + // Our latch was signalled. We should wake back up fully as we + // wil have some stuff to do. + if !latch.fall_asleep() { + self.logger.log(|| ThreadSleepInterruptedByLatch { + worker: worker_index, + latch_addr: latch.addr(), + }); + + idle_state.wake_fully(); + return; } - } - fn get_sleepy(&self, worker_index: usize) -> bool { loop { - // Acquire ordering suffices here. If some other worker - // was sleepy but no longer is, we will eventually see - // that, and until then it doesn't hurt to spin. - // Otherwise, we will do a compare-exchange which will - // assert a stronger order and acquire any reads etc that - // we must see. - let state = self.state.load(Ordering::Acquire); - if self.any_worker_is_sleepy(state) { - // somebody else is already sleepy, so we'll just wait our turn - debug_assert!( - !self.worker_is_sleepy(state, worker_index), - "worker {} called `is_sleepy()`, \ - but they are already sleepy (state={})", - worker_index, - state - ); - return false; - } else { - // make ourselves the sleepy one - let new_state = self.with_sleepy_worker(state, worker_index); - - // This must be SeqCst on success because we want to - // ensure: - // - // - That we observe any writes that preceded - // some prior tickle, and that tickle may have only - // done a SeqCst load on `self.state`. - // - That any subsequent tickle *definitely* sees this store. - // - // See the section on "Ensuring Sequentially - // Consistency" in README.md for more details. - // - // The failure ordering doesn't matter since we are - // about to spin around and do a fresh load. - if self - .state - .compare_exchange(state, new_state, Ordering::SeqCst, Ordering::Relaxed) - .is_ok() - { - return true; - } + let counters = self.counters.load(Ordering::SeqCst); + if counters.jobs_counter() > idle_state.sleepy_counter { + self.logger.log(|| ThreadSleepInterruptedByJob { + worker: worker_index, + }); + + // A new job was posted. We should return to just + // before the SLEEPY state so we can do another search + // and (if we fail to find work) go back to sleep. + idle_state.wake_partly(); + latch.wake_up(); + return; + } + + // Otherwise, let's move from IDLE to SLEEPING. + if self.counters.try_add_sleeping_thread(counters) { + break; } } + + // Successfully registered as asleep. + + self.logger.log(|| ThreadSleeping { + worker: worker_index, + latch_addr: latch.addr(), + }); + + // Flag ourselves as asleep and wait till we are notified. + // + // (Note that `is_blocked` is held under a mutex and the mutex + // was acquired *before* we incremented the "sleepy + // counter". This means that whomever is coming to wake us + // will have to wait until we release the mutex in the call to + // `wait`, so they will see this boolean as true.) + *is_blocked = true; + while *is_blocked { + is_blocked = sleep_state.condvar.wait(is_blocked).unwrap(); + } + + // Update other state: + idle_state.wake_fully(); + latch.wake_up(); + + self.logger.log(|| ThreadAwoken { + worker: worker_index, + latch_addr: latch.addr(), + }); + } - fn still_sleepy(&self, worker_index: usize) -> bool { - let state = self.state.load(Ordering::SeqCst); - self.worker_is_sleepy(state, worker_index) + /// Notify the given thread that it should wake up (if it is + /// sleeping). When this method is invoked, we typically know the + /// thread is asleep, though in rare cases it could have been + /// awoken by (e.g.) new work having been posted. + pub(super) fn notify_worker_latch_is_set(&self, target_worker_index: usize) { + self.wake_specific_thread(target_worker_index); } - fn sleep(&self, worker_index: usize) { + + /// Signals that `num_jobs` new jobs were injected into the thread + /// pool from outside. This function will ensure that there are + /// threads available to process them, waking threads from sleep + /// if necessary. + /// + /// # Parameters + /// + /// - `source_worker_index` -- index of the thread that did the + /// push, or `usize::MAX` if this came from outside the thread + /// pool -- it is used only for logging. + /// - `num_jobs` -- lower bound on number of jobs available for stealing. + /// We'll try to get at least one thread per job. + #[inline] + pub(super) fn new_injected_jobs( + &self, + source_worker_index: usize, + num_jobs: u32, + queue_was_empty: bool, + ) { + self.new_jobs(source_worker_index, num_jobs, queue_was_empty) + } + + /// Signals that `num_jobs` new jobs were pushed onto a thread's + /// local deque. This function will try to ensure that there are + /// threads available to process them, waking threads from sleep + /// if necessary. However, this is not guaranteed: under certain + /// race conditions, the function may fail to wake any new + /// threads; in that case the existing thread should eventually + /// pop the job. + /// + /// # Parameters + /// + /// - `source_worker_index` -- index of the thread that did the + /// push, or `usize::MAX` if this came from outside the thread + /// pool -- it is used only for logging. + /// - `num_jobs` -- lower bound on number of jobs available for stealing. + /// We'll try to get at least one thread per job. + #[inline] + pub(super) fn new_internal_jobs( + &self, + source_worker_index: usize, + num_jobs: u32, + queue_was_empty: bool, + ) { + self.new_jobs(source_worker_index, num_jobs, queue_was_empty) + } + + /// Common helper for `new_injected_jobs` and `new_internal_jobs`. + #[inline] + fn new_jobs( + &self, + source_worker_index: usize, + num_jobs: u32, + queue_was_empty: bool, + ) { + let mut counters = self.counters.load(Ordering::SeqCst); + + // If we find that the jobs counter is out of date, we have to fix that. + if counters.jobs_counter() != counters.sleepy_counter() { + self.sync_jobs_counter(&mut counters); + } + + let num_awake_but_idle = counters.awake_but_idle_threads(); + let num_sleepers = counters.sleeping_threads(); + + self.logger.log(|| JobThreadCounts { + worker: source_worker_index, + num_idle: num_awake_but_idle as u16, + num_sleepers: num_sleepers as u16, + }); + + if num_sleepers == 0 { + // nobody to wake + return; + } + + // Promote from u16 to u32 so we can interoperate with + // num_jobs more easily. + let num_awake_but_idle = num_awake_but_idle as u32; + let num_sleepers = num_sleepers as u32; + + // If the queue is non-empty, then we always wake up a worker + // -- clearly the existing idle jobs aren't enough. Otherwise, + // check to see if we have enough idle workers. + if !queue_was_empty { + let num_to_wake = std::cmp::min(num_jobs, num_sleepers); + self.wake_any_threads(num_to_wake); + } else if num_awake_but_idle < num_jobs { + let num_to_wake = std::cmp::min(num_jobs - num_awake_but_idle, num_sleepers); + self.wake_any_threads(num_to_wake); + } + } + + /// Invoked when we find that the "jobs counter" is not equal to + /// the "sleepy counter". This means that there may be threads + /// actively going to sleep. In that case, we have to synchronize + /// the jobs counter with the sleepy counter using a + /// compare-exchange. If it happens that this fails, then the + /// sleepy thread may have actually gone to sleep, so we re-load + /// the counters word. + #[cold] + fn sync_jobs_counter(&self, counters: &mut Counters) { loop { - // Acquire here suffices. If we observe that the current worker is still - // sleepy, then in fact we know that no writes have occurred, and anyhow - // we are going to do a CAS which will synchronize. - // - // If we observe that the state has changed, it must be - // due to a tickle, and then the Acquire means we also see - // any events that occured before that. - let state = self.state.load(Ordering::Acquire); - if self.worker_is_sleepy(state, worker_index) { - // It is important that we hold the lock when we do - // the CAS. Otherwise, if we were to CAS first, then - // the following sequence of events could occur: - // - // - Thread A (us) sets state to SLEEPING. - // - Thread B sets state to AWAKE. - // - Thread C sets state to SLEEPY(C). - // - Thread C sets state to SLEEPING. - // - Thread A reawakens, acquires lock, and goes to sleep. - // - // Now we missed the wake-up from thread B! But since - // we have the lock when we set the state to sleeping, - // that cannot happen. Note that the swap `tickle()` - // is not part of the lock, though, so let's play that - // out: - // - // # Scenario 1 - // - // - A loads state and see SLEEPY(A) - // - B swaps to AWAKE. - // - A locks, fails CAS - // - // # Scenario 2 - // - // - A loads state and see SLEEPY(A) - // - A locks, performs CAS - // - B swaps to AWAKE. - // - A waits (releasing lock) - // - B locks, notifies - // - // In general, acquiring the lock inside the loop - // seems like it could lead to bad performance, but - // actually it should be ok. This is because the only - // reason for the `compare_exchange` to fail is if an - // awaken comes, in which case the next cycle around - // the loop will just return. - let data = self.data.lock().unwrap(); - - // This must be SeqCst on success because we want to - // ensure: - // - // - That we observe any writes that preceded - // some prior tickle, and that tickle may have only - // done a SeqCst load on `self.state`. - // - That any subsequent tickle *definitely* sees this store. - // - // See the section on "Ensuring Sequentially - // Consistency" in README.md for more details. - // - // The failure ordering doesn't matter since we are - // about to spin around and do a fresh load. - if self - .state - .compare_exchange(state, SLEEPING, Ordering::SeqCst, Ordering::Relaxed) - .is_ok() - { - // Don't do this in a loop. If we do it in a loop, we need - // some way to distinguish the ABA scenario where the pool - // was awoken but before we could process it somebody went - // to sleep. Note that if we get a false wakeup it's not a - // problem for us, we'll just loop around and maybe get - // sleepy again. - self.logger.log(|| ThreadSleeping { - worker: worker_index - }); - drop(self.tickle.wait(data).unwrap()); - self.logger.log(|| ThreadAwoken { - worker: worker_index - }); - return; - } - } else { + if counters.jobs_counter() == counters.sleepy_counter() { return; } + + if self.counters.try_replicate_sleepy_counter(*counters) { + return; + } + + *counters = self.counters.load(Ordering::SeqCst); + } + } + + #[cold] + fn wake_any_threads( + &self, + mut num_to_wake: u32, + ) { + if num_to_wake > 0 { + for i in 0..self.worker_sleep_states.len() { + if self.wake_specific_thread(i) { + num_to_wake -= 1; + if num_to_wake == 0 { + return; + } + } + } + } + } + + fn wake_specific_thread(&self, index: usize) -> bool { + let sleep_state = &self.worker_sleep_states[index]; + + let mut is_blocked = sleep_state.is_blocked.lock().unwrap(); + if *is_blocked { + *is_blocked = false; + sleep_state.condvar.notify_one(); + + // When the thread went to sleep, it will have incremented + // this value. When we wake it, its our job to decrement + // it. We could have the thread do it, but that would + // introduce a delay between when the thread was + // *notified* and when this counter was decremented. That + // might mislead people with new work into thinking that + // there are sleeping threads that they should try to + // wake, when in fact there is nothing left for them to + // do. + self.counters.sub_sleeping_thread(); + + self.logger.log(|| ThreadNotify { + worker: index, + }); + + true + } else { + false } } } + +impl IdleState { + fn wake_fully(&mut self) { + self.rounds = 0; + self.sleepy_counter = INVALID_SLEEPY_COUNTER; + } + + fn wake_partly(&mut self) { + self.rounds = ROUNDS_UNTIL_SLEEPY; + self.sleepy_counter = INVALID_SLEEPY_COUNTER; + } +} + From 0b31522e49c6133afd67f20318009dbb06799f81 Mon Sep 17 00:00:00 2001 From: Niko Matsakis Date: Tue, 7 Apr 2020 14:47:02 -0400 Subject: [PATCH 07/33] document the new logging infrastructure more accurately --- rayon-core/src/log.rs | 27 +++++++++++++++++---------- 1 file changed, 17 insertions(+), 10 deletions(-) diff --git a/rayon-core/src/log.rs b/rayon-core/src/log.rs index 34eb9b9f9..bfa597b7c 100644 --- a/rayon-core/src/log.rs +++ b/rayon-core/src/log.rs @@ -1,11 +1,22 @@ //! Debug Logging //! -//! To use in a debug build, set the env var `RAYON_RS_LOG=1`. In a -//! release build, logs are compiled out by default unless Rayon is built -//! with `--cfg rayon_rs_log` (try `RUSTFLAGS="--cfg rayon_rs_log"`). +//! To use in a debug build, set the env var `RAYON_LOG` as +//! described below. In a release build, logs are compiled out by +//! default unless Rayon is built with `--cfg rayon_rs_log` (try +//! `RUSTFLAGS="--cfg rayon_rs_log"`). //! //! Note that logs are an internally debugging tool and their format //! is considered unstable, as are the details of how to enable them. +//! +//! # Valid values for RAYON_LOG +//! +//! The `RAYON_LOG` variable can take on the following values: +//! +//! * `tail:` -- dumps the last 10,000 events into the given file; +//! useful for tracking down deadlocks +//! * `profile:` -- dumps only those events needed to reconstruct how +//! many workers are active at a given time +//! * `all:` -- dumps every event to the file; useful for debugging use crossbeam_channel::{self, Receiver, Sender}; use std::collections::VecDeque; @@ -107,12 +118,8 @@ impl Logger { return Self::disabled(); } - // format: - // - // tail: -- dumps the last 10,000 events - // profile: -- dumps every Nth event - // all: -- dumps every event to the file - let env_log = match env::var("RAYON_RS_LOG") { + // see the doc comment for the format + let env_log = match env::var("RAYON_LOG") { Ok(s) => s, Err(_) => return Self::disabled(), }; @@ -134,7 +141,7 @@ impl Logger { Self::profile_logger_thread(num_workers, filename, 10_000, receiver) }); } else { - panic!("RAYON_RS_LOG should be 'tail:' or 'profile:'"); + panic!("RAYON_LOG should be 'tail:' or 'profile:'"); } return Logger { From 07137531ba0443dfababda1f347b8bdecdc23fa1 Mon Sep 17 00:00:00 2001 From: Josh Stone Date: Wed, 11 Mar 2020 10:46:51 -0700 Subject: [PATCH 08/33] Read all SpinLatch fields before we call set() --- rayon-core/src/latch.rs | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/rayon-core/src/latch.rs b/rayon-core/src/latch.rs index 9d7d9cb93..989ff5930 100644 --- a/rayon-core/src/latch.rs +++ b/rayon-core/src/latch.rs @@ -161,8 +161,12 @@ impl<'r> AsCoreLatch for SpinLatch<'r> { impl<'r> Latch for SpinLatch<'r> { #[inline] fn set(&self) { - if self.core_latch.set() { - self.registry.notify_worker_latch_is_set(self.target_worker_index); + // NB: once we `set`, the target may proceed and invalidate `&self`! + let SpinLatch { ref core_latch, registry, target_worker_index } = *self; + if core_latch.set() { + // FIXME: we probably need to ensure the registry is still alive too, + // specifically for the case of cross-pool notifications. + registry.notify_worker_latch_is_set(target_worker_index); } } } From cbd484ca6d65a6ce9156ea5510d68b2990b0a2e1 Mon Sep 17 00:00:00 2001 From: Josh Stone Date: Wed, 11 Mar 2020 14:29:18 -0700 Subject: [PATCH 09/33] Keep the registry alive for a cross-pool SpinLatch --- rayon-core/src/latch.rs | 33 ++++++++++++++++++++++++++------- rayon-core/src/registry.rs | 2 +- 2 files changed, 27 insertions(+), 8 deletions(-) diff --git a/rayon-core/src/latch.rs b/rayon-core/src/latch.rs index 989ff5930..c6a47864f 100644 --- a/rayon-core/src/latch.rs +++ b/rayon-core/src/latch.rs @@ -1,5 +1,5 @@ use std::sync::atomic::{AtomicUsize, Ordering}; -use std::sync::{Condvar, Mutex}; +use std::sync::{Arc, Condvar, Mutex}; use std::usize; use crate::registry::{Registry, WorkerThread}; @@ -128,8 +128,9 @@ impl CoreLatch { /// that becomes true when `set()` is called. pub(super) struct SpinLatch<'r> { core_latch: CoreLatch, - registry: &'r Registry, + registry: &'r Arc, target_worker_index: usize, + cross: bool, } impl<'r> SpinLatch<'r> { @@ -143,6 +144,17 @@ impl<'r> SpinLatch<'r> { core_latch: CoreLatch::new(), registry: thread.registry(), target_worker_index: thread.index(), + cross: false, + } + } + + /// Creates a new spin latch for cross-threadpool blocking. Notably, we + /// need to make sure the registry is kept alive after setting, so we can + /// safely call the notification. + pub(super) fn cross(thread: &'r WorkerThread) -> SpinLatch { + SpinLatch { + cross: true, + ..SpinLatch::new(thread) } } @@ -161,11 +173,18 @@ impl<'r> AsCoreLatch for SpinLatch<'r> { impl<'r> Latch for SpinLatch<'r> { #[inline] fn set(&self) { - // NB: once we `set`, the target may proceed and invalidate `&self`! - let SpinLatch { ref core_latch, registry, target_worker_index } = *self; - if core_latch.set() { - // FIXME: we probably need to ensure the registry is still alive too, - // specifically for the case of cross-pool notifications. + let cross_registry; + let registry = if self.cross { + // Ensure the registry stays alive while we notify it. + cross_registry = Arc::clone(self.registry); + &cross_registry + } else { + self.registry + }; + let target_worker_index = self.target_worker_index; + + // NOTE: Once we `set`, the target may proceed and invalidate `&self`! + if self.core_latch.set() { registry.notify_worker_latch_is_set(target_worker_index); } } diff --git a/rayon-core/src/registry.rs b/rayon-core/src/registry.rs index 037013c72..1b26e2874 100644 --- a/rayon-core/src/registry.rs +++ b/rayon-core/src/registry.rs @@ -480,7 +480,7 @@ impl Registry { // This thread is a member of a different pool, so let it process // other work while waiting for this `op` to complete. debug_assert!(current_thread.registry().id() != self.id()); - let latch = SpinLatch::new(current_thread); + let latch = SpinLatch::cross(current_thread); let job = StackJob::new( |injected| { let worker_thread = WorkerThread::current(); From 90da62d078248f2584fd2c2b07ae5d642bb02036 Mon Sep 17 00:00:00 2001 From: Niko Matsakis Date: Tue, 7 Apr 2020 14:54:29 -0400 Subject: [PATCH 10/33] apply cargo fmt --- rayon-core/src/latch.rs | 11 +++----- rayon-core/src/log.rs | 17 +++-------- rayon-core/src/registry.rs | 33 ++++++++++++---------- rayon-core/src/scope/mod.rs | 6 ++-- rayon-core/src/sleep/counters.rs | 17 ++++++----- rayon-core/src/sleep/mod.rs | 48 ++++++++++++++------------------ 6 files changed, 60 insertions(+), 72 deletions(-) diff --git a/rayon-core/src/latch.rs b/rayon-core/src/latch.rs index c6a47864f..e98168a7c 100644 --- a/rayon-core/src/latch.rs +++ b/rayon-core/src/latch.rs @@ -78,8 +78,7 @@ impl CoreLatch { /// if the owning thread may proceed to fall asleep, false if the /// latch was set in the meantime. pub(super) fn get_sleepy(&self) -> bool { - self - .state + self.state .compare_exchange(UNSET, SLEEPY, Ordering::SeqCst, Ordering::Relaxed) .is_ok() } @@ -88,8 +87,7 @@ impl CoreLatch { /// true if the owning thread should block, or false if the latch /// was set in the meantime. pub(super) fn fall_asleep(&self) -> bool { - self - .state + self.state .compare_exchange(SLEEPY, SLEEPING, Ordering::SeqCst, Ordering::Relaxed) .is_ok() } @@ -100,9 +98,8 @@ impl CoreLatch { pub(super) fn wake_up(&self) { if !self.probe() { let _ = - self - .state - .compare_exchange(SLEEPING, UNSET, Ordering::SeqCst, Ordering::Relaxed); + self.state + .compare_exchange(SLEEPING, UNSET, Ordering::SeqCst, Ordering::Relaxed); } } diff --git a/rayon-core/src/log.rs b/rayon-core/src/log.rs index bfa597b7c..cf073f355 100644 --- a/rayon-core/src/log.rs +++ b/rayon-core/src/log.rs @@ -39,9 +39,7 @@ pub(super) enum Event { }, /// Indicates that a worker thread started execution. - ThreadTerminate { - worker: usize, - }, + ThreadTerminate { worker: usize }, /// Indicates that a worker thread became idle, blocked on `latch_addr`. ThreadIdle { worker: usize, latch_addr: usize }, @@ -132,9 +130,7 @@ impl Logger { Self::tail_logger_thread(num_workers, filename, 10_000, receiver) }); } else if env_log == "all" { - ::std::thread::spawn(move || { - Self::all_logger_thread(num_workers, receiver) - }); + ::std::thread::spawn(move || Self::all_logger_thread(num_workers, receiver)); } else if env_log.starts_with("profile:") { let filename = env_log["profile:".len()..].to_string(); ::std::thread::spawn(move || { @@ -160,9 +156,7 @@ impl Logger { } if let Some(sender) = &self.sender { - sender - .send(event()) - .unwrap(); + sender.send(event()).unwrap(); } } @@ -264,10 +258,7 @@ impl Logger { } } - fn all_logger_thread( - num_workers: usize, - receiver: Receiver, - ) { + fn all_logger_thread(num_workers: usize, receiver: Receiver) { let stderr = std::io::stderr(); let mut state = SimulatorState::new(num_workers); diff --git a/rayon-core/src/registry.rs b/rayon-core/src/registry.rs index 1b26e2874..bbcd8b86d 100644 --- a/rayon-core/src/registry.rs +++ b/rayon-core/src/registry.rs @@ -1,7 +1,7 @@ use crate::job::{JobFifo, JobRef, StackJob}; use crate::latch::{AsCoreLatch, CoreLatch, CountLatch, Latch, LockLatch, SpinLatch}; -use crate::log::Logger; use crate::log::Event::*; +use crate::log::Logger; use crate::sleep::Sleep; use crate::unwind; use crate::util::leak; @@ -384,7 +384,7 @@ impl Registry { /// you are not on a worker of this registry. pub(super) fn inject(&self, injected_jobs: &[JobRef]) { self.log(|| JobsInjected { - count: injected_jobs.len() + count: injected_jobs.len(), }); // It should not be possible for `state.terminate` to be true @@ -404,14 +404,15 @@ impl Registry { self.injected_jobs.push(job_ref); } - self.sleep.new_injected_jobs(usize::MAX, injected_jobs.len() as u32, queue_was_empty); + self.sleep + .new_injected_jobs(usize::MAX, injected_jobs.len() as u32, queue_was_empty); } fn pop_injected_job(&self, worker_index: usize) -> Option { let job = self.injected_jobs.pop().ok(); if job.is_some() { self.log(|| JobUninjected { - worker: worker_index + worker: worker_index, }); } job @@ -517,7 +518,10 @@ impl Registry { pub(super) fn increment_terminate_count(&self) { let previous = self.terminate_count.fetch_add(1, Ordering::AcqRel); debug_assert!(previous != 0, "registry ref count incremented from zero"); - assert!(previous != std::usize::MAX, "overflow in registry ref count"); + assert!( + previous != std::usize::MAX, + "overflow in registry ref count" + ); } /// Signals that the thread-pool which owns this registry has been @@ -652,7 +656,9 @@ impl WorkerThread { self.log(|| JobPushed { worker: self.index }); let queue_was_empty = self.worker.is_empty(); self.worker.push(job); - self.registry.sleep.new_internal_jobs(self.index, 1, queue_was_empty); + self.registry + .sleep + .new_internal_jobs(self.index, 1, queue_was_empty); } #[inline] @@ -674,9 +680,7 @@ impl WorkerThread { let popped_job = self.worker.pop(); if popped_job.is_some() { - self.log(|| JobPopped { - worker: self.index - }); + self.log(|| JobPopped { worker: self.index }); } popped_job @@ -726,7 +730,10 @@ impl WorkerThread { // wait. self.registry.sleep.work_found(idle_state); - self.log(|| ThreadSawLatchSet { worker: self.index, latch_addr: latch.addr() }); + self.log(|| ThreadSawLatchSet { + worker: self.index, + latch_addr: latch.addr(), + }); mem::forget(abort_guard); // successful execution, do not abort } @@ -760,7 +767,7 @@ impl WorkerThread { Steal::Success(d) => { self.log(|| JobStolen { worker: self.index, - victim: victim_index + victim: victim_index, }); return Some(d); } @@ -819,9 +826,7 @@ unsafe fn main_loop(worker: Worker, registry: Arc, index: usiz // Normal termination, do not abort. mem::forget(abort_guard); - worker_thread.log(|| ThreadTerminate { - worker: index, - }); + worker_thread.log(|| ThreadTerminate { worker: index }); // Inform a user callback that we exited a thread. if let Some(ref handler) = registry.exit_handler { diff --git a/rayon-core/src/scope/mod.rs b/rayon-core/src/scope/mod.rs index 296a37f35..7d274920b 100644 --- a/rayon-core/src/scope/mod.rs +++ b/rayon-core/src/scope/mod.rs @@ -582,11 +582,13 @@ impl<'scope> ScopeBase<'scope> { mem::forget(err); // ownership now transferred into self.panic } - self.job_completed_latch.set_and_tickle_one(&self.registry, self.owner_thread_index); + self.job_completed_latch + .set_and_tickle_one(&self.registry, self.owner_thread_index); } unsafe fn job_completed_ok(&self) { - self.job_completed_latch.set_and_tickle_one(&self.registry, self.owner_thread_index); + self.job_completed_latch + .set_and_tickle_one(&self.registry, self.owner_thread_index); } unsafe fn steal_till_jobs_complete(&self, owner_thread: &WorkerThread) { diff --git a/rayon-core/src/sleep/counters.rs b/rayon-core/src/sleep/counters.rs index 5fc4e5cf4..15046b85c 100644 --- a/rayon-core/src/sleep/counters.rs +++ b/rayon-core/src/sleep/counters.rs @@ -11,12 +11,12 @@ pub(super) struct AtomicCounters { /// - The bits 0x4444 are the number of **sleeping threads**. /// /// See the struct `Counters` below. - value: AtomicU64 + value: AtomicU64, } #[derive(Copy, Clone)] pub(super) struct Counters { - word: u64 + word: u64, } #[derive(Copy, Clone, Debug, PartialEq, PartialOrd)] @@ -40,7 +40,9 @@ pub(super) const ZERO_SLEEPY_COUNTER: SleepyCounter = SleepyCounter(0); impl AtomicCounters { pub(super) fn new() -> AtomicCounters { - AtomicCounters { value: AtomicU64::new(0) } + AtomicCounters { + value: AtomicU64::new(0), + } } pub(super) fn load(&self, ordering: Ordering) -> Counters { @@ -49,12 +51,9 @@ impl AtomicCounters { #[inline] fn try_exchange(&self, old_value: Counters, new_value: Counters, ordering: Ordering) -> bool { - self.value.compare_exchange( - old_value.word, - new_value.word, - ordering, - Ordering::Relaxed, - ).is_ok() + self.value + .compare_exchange(old_value.word, new_value.word, ordering, Ordering::Relaxed) + .is_ok() } /// Adds an idle thread. This cannot fail. diff --git a/rayon-core/src/sleep/mod.rs b/rayon-core/src/sleep/mod.rs index 87b498531..7202608d9 100644 --- a/rayon-core/src/sleep/mod.rs +++ b/rayon-core/src/sleep/mod.rs @@ -1,17 +1,19 @@ //! Code that decides when workers should go to sleep. See README.md //! for an overview. -use crossbeam_utils::CachePadded; use crate::latch::CoreLatch; -use crate::log::Logger; use crate::log::Event::*; +use crate::log::Logger; +use crossbeam_utils::CachePadded; use std::sync::atomic::Ordering; use std::sync::{Condvar, Mutex}; use std::thread; use std::usize; mod counters; -use self::counters::{AtomicCounters, Counters, INVALID_SLEEPY_COUNTER, SleepyCounter, ZERO_SLEEPY_COUNTER}; +use self::counters::{ + AtomicCounters, Counters, SleepyCounter, INVALID_SLEEPY_COUNTER, ZERO_SLEEPY_COUNTER, +}; pub(super) struct Sleep { logger: Logger, @@ -92,11 +94,7 @@ impl Sleep { } #[inline] - pub(super) fn no_work_found( - &self, - idle_state: &mut IdleState, - latch: &CoreLatch, - ) { + pub(super) fn no_work_found(&self, idle_state: &mut IdleState, latch: &CoreLatch) { if idle_state.rounds < ROUNDS_UNTIL_SLEEPY { thread::yield_now(); idle_state.rounds += 1; @@ -119,13 +117,22 @@ impl Sleep { let counters = self.counters.load(Ordering::Relaxed); let sleepy_counter = counters.sleepy_counter(); if sleepy_counter.is_max() { - if self.counters.try_rollover_jobs_and_sleepy_counters(counters) { - self.logger.log(|| ThreadSleepy { worker: worker_index, sleepy_counter: 0 }); + if self + .counters + .try_rollover_jobs_and_sleepy_counters(counters) + { + self.logger.log(|| ThreadSleepy { + worker: worker_index, + sleepy_counter: 0, + }); return ZERO_SLEEPY_COUNTER; } } else { if self.counters.try_add_sleepy_thread(counters) { - self.logger.log(|| ThreadSleepy { worker: worker_index, sleepy_counter: sleepy_counter.as_u16() }); + self.logger.log(|| ThreadSleepy { + worker: worker_index, + sleepy_counter: sleepy_counter.as_u16(), + }); return sleepy_counter; } } @@ -209,7 +216,6 @@ impl Sleep { worker: worker_index, latch_addr: latch.addr(), }); - } /// Notify the given thread that it should wake up (if it is @@ -220,7 +226,6 @@ impl Sleep { self.wake_specific_thread(target_worker_index); } - /// Signals that `num_jobs` new jobs were injected into the thread /// pool from outside. This function will ensure that there are /// threads available to process them, waking threads from sleep @@ -270,12 +275,7 @@ impl Sleep { /// Common helper for `new_injected_jobs` and `new_internal_jobs`. #[inline] - fn new_jobs( - &self, - source_worker_index: usize, - num_jobs: u32, - queue_was_empty: bool, - ) { + fn new_jobs(&self, source_worker_index: usize, num_jobs: u32, queue_was_empty: bool) { let mut counters = self.counters.load(Ordering::SeqCst); // If we find that the jobs counter is out of date, we have to fix that. @@ -337,10 +337,7 @@ impl Sleep { } #[cold] - fn wake_any_threads( - &self, - mut num_to_wake: u32, - ) { + fn wake_any_threads(&self, mut num_to_wake: u32) { if num_to_wake > 0 { for i in 0..self.worker_sleep_states.len() { if self.wake_specific_thread(i) { @@ -372,9 +369,7 @@ impl Sleep { // do. self.counters.sub_sleeping_thread(); - self.logger.log(|| ThreadNotify { - worker: index, - }); + self.logger.log(|| ThreadNotify { worker: index }); true } else { @@ -394,4 +389,3 @@ impl IdleState { self.sleepy_counter = INVALID_SLEEPY_COUNTER; } } - From 30f592834a2c34a871c0fffb12b1c1fa0580b5e7 Mon Sep 17 00:00:00 2001 From: Niko Matsakis Date: Tue, 4 Aug 2020 06:01:53 -0400 Subject: [PATCH 11/33] add missing lifetime annotation --- rayon-core/src/latch.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rayon-core/src/latch.rs b/rayon-core/src/latch.rs index e98168a7c..c4883a089 100644 --- a/rayon-core/src/latch.rs +++ b/rayon-core/src/latch.rs @@ -148,7 +148,7 @@ impl<'r> SpinLatch<'r> { /// Creates a new spin latch for cross-threadpool blocking. Notably, we /// need to make sure the registry is kept alive after setting, so we can /// safely call the notification. - pub(super) fn cross(thread: &'r WorkerThread) -> SpinLatch { + pub(super) fn cross(thread: &'r WorkerThread) -> SpinLatch<'r> { SpinLatch { cross: true, ..SpinLatch::new(thread) From b8017a3b58b116fdde73969f07886aaf06433c34 Mon Sep 17 00:00:00 2001 From: Josh Stone Date: Wed, 8 Apr 2020 11:13:06 -0700 Subject: [PATCH 12/33] update to crossbeam-channel 0.4 --- ci/compat-Cargo.lock | 33 +++++++++++++++++---------------- rayon-core/Cargo.toml | 2 +- 2 files changed, 18 insertions(+), 17 deletions(-) diff --git a/ci/compat-Cargo.lock b/ci/compat-Cargo.lock index bd0456e5e..9d655ddc3 100644 --- a/ci/compat-Cargo.lock +++ b/ci/compat-Cargo.lock @@ -187,6 +187,15 @@ dependencies = [ "objc 0.2.7 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "crossbeam-channel" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "crossbeam-utils 0.7.2 (registry+https://github.com/rust-lang/crates.io-index)", + "maybe-uninit 2.0.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "crossbeam-deque" version = "0.7.3" @@ -225,16 +234,6 @@ dependencies = [ name = "crossbeam-utils" version = "0.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "autocfg 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)", - "cfg-if 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)", - "lazy_static 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "derivative" -version = "2.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "proc-macro2 1.0.18 (registry+https://github.com/rust-lang/crates.io-index)", "quote 1.0.7 (registry+https://github.com/rust-lang/crates.io-index)", @@ -1040,6 +1039,7 @@ dependencies = [ name = "rayon-core" version = "1.7.1" dependencies = [ + "crossbeam-channel 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)", "crossbeam-deque 0.7.3 (registry+https://github.com/rust-lang/crates.io-index)", "crossbeam-queue 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)", "crossbeam-utils 0.7.2 (registry+https://github.com/rust-lang/crates.io-index)", @@ -1439,14 +1439,15 @@ source = "registry+https://github.com/rust-lang/crates.io-index" "checksum cgl 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)" = "0ced0551234e87afee12411d535648dd89d2e7f34c78b753395567aff3d447ff" "checksum cgmath 0.17.0 (registry+https://github.com/rust-lang/crates.io-index)" = "283944cdecc44bf0b8dd010ec9af888d3b4f142844fdbe026c20ef68148d6fe7" "checksum cloudabi 0.0.3 (registry+https://github.com/rust-lang/crates.io-index)" = "ddfc5b9aa5d4507acaf872de71051dfd0e309860e88966e1051e462a077aac4f" -"checksum cocoa 0.20.1 (registry+https://github.com/rust-lang/crates.io-index)" = "8f7b6f3f7f4f0b3ec5c5039aaa9e8c3cef97a7a480a400fd62944841314f293d" -"checksum core-foundation 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)" = "57d24c7a13c43e870e37c1556b74555437870a04514f7685f5b354e090567171" -"checksum core-foundation-sys 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)" = "b3a71ab494c0b5b860bdc8407ae08978052417070c2ced38573a9157ad75b8ac" -"checksum core-graphics 0.19.0 (registry+https://github.com/rust-lang/crates.io-index)" = "59e78b2e0aaf43f08e7ae0d6bc96895ef72ff0921c7d4ff4762201b2dba376dd" -"checksum core-video-sys 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)" = "34ecad23610ad9757664d644e369246edde1803fcb43ed72876565098a5d3828" +"checksum cocoa 0.19.1 (registry+https://github.com/rust-lang/crates.io-index)" = "f29f7768b2d1be17b96158e3285951d366b40211320fb30826a76cb7a0da6400" +"checksum core-foundation 0.6.4 (registry+https://github.com/rust-lang/crates.io-index)" = "25b9e03f145fd4f2bf705e07b900cd41fc636598fe5dc452fd0db1441c3f496d" +"checksum core-foundation-sys 0.6.2 (registry+https://github.com/rust-lang/crates.io-index)" = "e7ca8a5221364ef15ce201e8ed2f609fc312682a8f4e0e3d4aa5879764e0fa3b" +"checksum core-graphics 0.17.3 (registry+https://github.com/rust-lang/crates.io-index)" = "56790968ab1c8a1202a102e6de05fc6e1ec87da99e4e93e9a7d13efbfc1e95a9" +"checksum core-video-sys 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)" = "8dc065219542086f72d1e9f7aadbbab0989e980263695d129d502082d063a9d0" +"checksum crossbeam-channel 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)" = "cced8691919c02aac3cb0a1bc2e9b73d89e832bf9a06fc579d4e71b68a2da061" "checksum crossbeam-deque 0.7.3 (registry+https://github.com/rust-lang/crates.io-index)" = "9f02af974daeee82218205558e51ec8768b48cf524bd01d550abe5573a608285" "checksum crossbeam-epoch 0.8.2 (registry+https://github.com/rust-lang/crates.io-index)" = "058ed274caafc1f60c4997b5fc07bf7dc7cca454af7c6e81edffe5f33f70dace" -"checksum crossbeam-queue 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)" = "774ba60a54c213d409d5353bda12d49cd68d14e45036a285234c8d6f91f92570" +"checksum crossbeam-queue 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "c695eeca1e7173472a32221542ae469b3e9aac3a4fc81f7696bcad82029493db" "checksum crossbeam-utils 0.7.2 (registry+https://github.com/rust-lang/crates.io-index)" = "c3c7c73a2d1e9fc0886a08b93e98eb643461230d5f1925e4036204d5f2e261a8" "checksum derivative 2.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "cb582b60359da160a9477ee80f15c8d784c477e69c217ef2cdd4169c24ea380f" "checksum dispatch 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "bd0c93bb4b0c6d9b77f4435b0ae98c24d17f1c45b2ff844c6151a07256ca923b" diff --git a/rayon-core/Cargo.toml b/rayon-core/Cargo.toml index b731baeea..fd72d1263 100644 --- a/rayon-core/Cargo.toml +++ b/rayon-core/Cargo.toml @@ -18,7 +18,7 @@ categories = ["concurrency"] [dependencies] num_cpus = "1.2" lazy_static = "1" -crossbeam-channel = "0.3.9" +crossbeam-channel = "0.4.2" crossbeam-deque = "0.7.2" crossbeam-queue = "0.2" crossbeam-utils = "0.7" From e2fc62e92a3807ba7ecee47efdee3322658a60e8 Mon Sep 17 00:00:00 2001 From: Niko Matsakis Date: Fri, 10 Apr 2020 05:58:47 -0400 Subject: [PATCH 13/33] Update rayon-core/src/registry.rs Co-Authored-By: Josh Stone --- rayon-core/src/registry.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rayon-core/src/registry.rs b/rayon-core/src/registry.rs index bbcd8b86d..40c752369 100644 --- a/rayon-core/src/registry.rs +++ b/rayon-core/src/registry.rs @@ -535,7 +535,7 @@ impl Registry { } } - /// Notify the worker that the latch they are sleeping on has bene "set". + /// Notify the worker that the latch they are sleeping on has been "set". pub(super) fn notify_worker_latch_is_set(&self, target_worker_index: usize) { self.sleep.notify_worker_latch_is_set(target_worker_index); } From c554e949efdea5cbf66a5c37fb85f8da7936e105 Mon Sep 17 00:00:00 2001 From: Niko Matsakis Date: Fri, 10 Apr 2020 06:07:33 -0400 Subject: [PATCH 14/33] add warning comments --- rayon-core/src/latch.rs | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/rayon-core/src/latch.rs b/rayon-core/src/latch.rs index c4883a089..1e03210cf 100644 --- a/rayon-core/src/latch.rs +++ b/rayon-core/src/latch.rs @@ -32,6 +32,14 @@ use crate::registry::{Registry, WorkerThread}; /// README](/src/sleep/README.md#tickle-then-get-sleepy) for details. pub(super) trait Latch { /// Set the latch, signalling others. + /// + /// # WARNING + /// + /// Setting a latch triggers other threads to wake up and (in some + /// cases) complete. This may, in turn, cause memory to be + /// allocated and so forth. One must be very careful about this, + /// and it's typically better to read all the fields you will need + /// to access *before* a latch is set! fn set(&self); } From 0253df041d2425c6e4746a177ab4d7e262cef9c1 Mon Sep 17 00:00:00 2001 From: Niko Matsakis Date: Fri, 10 Apr 2020 06:16:23 -0400 Subject: [PATCH 15/33] add some comments explaining what is going on in a bit more depth. --- rayon-core/src/latch.rs | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/rayon-core/src/latch.rs b/rayon-core/src/latch.rs index 1e03210cf..f933a0b98 100644 --- a/rayon-core/src/latch.rs +++ b/rayon-core/src/latch.rs @@ -179,17 +179,29 @@ impl<'r> Latch for SpinLatch<'r> { #[inline] fn set(&self) { let cross_registry; + let registry = if self.cross { // Ensure the registry stays alive while we notify it. + // Otherwise, it would be possible that we set the spin + // latch and the other thread sees it and exits, causing + // the registry to be deallocated, all before we get a + // chance to invoke `registry.notify_worker_latch_is_set`. cross_registry = Arc::clone(self.registry); &cross_registry } else { + // If this is not a "cross-registry" spin-latch, then the + // thread which is performing `set` is itself ensuring + // that the registry stays alive. self.registry }; let target_worker_index = self.target_worker_index; // NOTE: Once we `set`, the target may proceed and invalidate `&self`! if self.core_latch.set() { + // Subtle: at this point, we can no longer read from + // `self`, because the thread owning this spin latch may + // have awoken and deallocated the latch. Therefore, we + // only use fields whose values we already read. registry.notify_worker_latch_is_set(target_worker_index); } } From 4619d34e8df15e3cd1fb45f0b34ec0319315a84c Mon Sep 17 00:00:00 2001 From: Niko Matsakis Date: Fri, 10 Apr 2020 07:33:30 -0400 Subject: [PATCH 16/33] [WIP] start adding comments --- rayon-core/src/sleep/README.md | 516 ++++++++----------------------- rayon-core/src/sleep/counters.rs | 44 +-- rayon-core/src/sleep/mod.rs | 13 +- 3 files changed, 166 insertions(+), 407 deletions(-) diff --git a/rayon-core/src/sleep/README.md b/rayon-core/src/sleep/README.md index bc2af869f..867496fa8 100644 --- a/rayon-core/src/sleep/README.md +++ b/rayon-core/src/sleep/README.md @@ -1,388 +1,134 @@ # Introduction: the sleep module The code in this module governs when worker threads should go to -sleep. This is a tricky topic -- the work-stealing algorithm relies on -having active worker threads running around stealing from one -another. But, if there isn't a lot of work, this can be a bit of a -drag, because it requires high CPU usage. - -The code in this module takes a fairly simple approach to the -problem. It allows worker threads to fall asleep if they have failed -to steal work after various thresholds; however, whenever new work -appears, they will wake up briefly and try to steal again. There are some -shortcomings in this current approach: - -- it can (to some extent) scale *down* the amount of threads, but they - can never be scaled *up*. The latter might be useful in the case of - user tasks that must (perhaps very occasionally and unpredictably) - block for long periods of time. - - however, the preferred approach to this is for users to adopt futures - instead (and indeed this sleeping work is intended to enable future - integration). -- we have no way to wake up threads in a fine-grained or targeted - manner. The current system wakes up *all* sleeping threads whenever - *any* of them might be interested in an event. This means that while - we can scale CPU usage down, we do is in a fairly "bursty" manner, - where everyone comes online, then some of them go back offline. - -# The interface for workers - -Workers interact with the sleep module by invoking three methods: - -- `work_found()`: signals that the worker found some work and is about - to execute it. -- `no_work_found()`: signals that the worker searched all available sources - for work and found none. - - It is important for the coherence of the algorithm that if work - was available **before the search started**, it would have been - found. If work was made available during the search, then it's ok that - it might have been overlooked. -- `tickle()`: indicates that new work is available (e.g., a job has - been pushed to the local deque) or that some other blocking - condition has been resolved (e.g., a latch has been set). Wakes up any - sleeping workers. - -When in a loop searching for work, Workers also have to maintain an -integer `yields` that they provide to the `sleep` module (which will -return a new value for the next time). Thus the basic worker "find -work" loop looks like this (this is `wait_until()`, basically): - -```rust -let mut yields = 0; -while /* not done */ { - if let Some(job) = search_for_work() { - yields = work_found(self.index, yields); - } else { - yields = no_work_found(self.index, yields); - } -} -``` - -# Getting sleepy and falling asleep - -The basic idea here is that every worker goes through three states: - -- **Awake:** actively hunting for tasks. -- **Sleepy:** still actively hunting for tasks, but we have signaled that - we might go to sleep soon if we don't find any. -- **Asleep:** actually asleep (blocked on a condition variable). - -At any given time, only **one** worker can be in the sleepy -state. This allows us to coordinate the entire sleep protocol using a -single `AtomicUsize` and without the need of epoch counters or other -things that might rollover and so forth. - -Whenever a worker invokes `work_found()`, it transitions back to the -**awake** state. In other words, if it was sleepy, it stops being -sleepy. (`work_found()` cannot be invoked when the worker is asleep, -since then it is not doing anything.) - -On the other hand, whenever a worker invokes `no_work_found()`, it -*may* transition to a more sleepy state. To track this, we use the -counter `yields` that is maintained by the worker's steal loop. This -counter starts at 0. Whenever work is found, the counter is returned -to 0. But each time that **no** work is found, the counter is -incremented. Eventually it will reach a threshold -`ROUNDS_UNTIL_SLEEPY`. At this point, the worker will try to become -the sleepy one. It does this by executing a CAS into the global -registry state (details on this below). If that attempt is successful, -then the counter is incremented again, so that it is equal to -`ROUNDS_UNTIL_SLEEPY + 1`. Otherwise, the counter stays the same (and -hence we will keep trying to become sleepy until either work is found -or we are successful). - -Becoming sleepy does not put us to sleep immediately. Instead, we keep -iterating and looking for work for some further number of rounds. If -during this search we **do** find work, then we will return the -counter to 0 and also reset the global state to indicate we are no -longer sleepy. - -But if again no work is found, `yields` will eventually reach the -value `ROUNDS_UNTIL_ASLEEP`. At that point, we will try to transition -from **sleepy** to **asleep**. This is done by the helper fn -`sleep()`, which executes another CAS on the global state that removes -our worker as the sleepy worker and instead sets a flag to indicate -that there are sleeping workers present (the flag may already have -been set, that's ok). Assuming that CAS succeeds, we will block on a -condition variable. - -# Tickling workers - -Of course, while all the stuff in the previous section is happening, -other workers are (hopefully) producing new work. There are three kinds of -events that can allow a blocked worker to make progress: - -1. A new task is pushed onto a worker's deque. This task could be stolen. -2. A new task is injected into the thread-pool from the outside. This - task could be uninjected and executed. -3. A latch is set. One of the sleeping workers might have been waiting for - that before it could go on. - -Whenever one of these things happens, the worker (or thread, more generally) -responsible must invoke `tickle()`. Tickle will basically wake up **all** -the workers: - -- If any worker was the sleepy one, then the global state is changed - so that there is no sleepy worker. The sleepy one will notice this - when it next invokes `no_work_found()` and return to the *awake* state - (with a yield counter of zero). -- If any workers were actually **asleep**, then we invoke - `notify_all()` on the condition variable, which will cause them to - awaken and start over from the awake state (with a yield counter of - zero). - -Because `tickle()` is invoked very frequently -- and hopefully most of -the time it is not needed, because the workers are already actively -stealing -- it is important that it be very cheap. The current design -requires, in the case where nobody is even sleepy, just a load and a -compare. If there are sleepy workers, a swap is needed. If there -workers *asleep*, we must naturally acquire the lock and signal the -condition variable. - -# The global state - -We manage all of the above state transitions using a small bit of global -state (well, global to the registry). This is stored in the `Sleep` struct. -The primary thing is a single `AtomicUsize`. The value in this usize packs -in two pieces of information: - -1. **Are any workers asleep?** This is just one bit (yes or no). -2. **Which worker is the sleepy worker, if any?** This is a worker id. - -We use bit 0 to indicate whether any workers are asleep. So if `state -& 1` is zero, then no workers are sleeping. But if `state & 1` is 1, -then some workers are either sleeping or on their way to falling -asleep (i.e., they have acquired the lock). - -The remaining bits are used to store if there is a sleepy worker. We -want `0` to indicate that there is no sleepy worker. If there a sleepy -worker with index `worker_index`, we would store `(worker_index + 1) -<< 1` . The `+1` is there because worker indices are 0-based, so this -ensures that the value is non-zero, and the shift skips over the -sleepy bit. - -Some examples: - -- `0`: everyone is awake, nobody is sleepy -- `1`: some workers are asleep, no sleepy worker -- `2`: no workers are asleep, but worker 0 is sleepy (`(0 + 1) << 1 == 2`). -- `3`: some workers are asleep, and worker 0 is sleepy. - -# Correctness level 1: avoiding deadlocks etc - -In general, we do not want to miss wakeups. Two bad things could happen: - -- **Suboptimal performance**: If this is a wakeup about a new job being - pushed into a local deque, it won't deadlock, but it will cause - things to run slowly. The reason that it won't deadlock is that we - know at least one thread is active (the one doing the pushing), and - it will (sooner or later) try to pop this item from its own local - deque. -- **Deadlocks:** If this is a wakeup about an injected job or a latch that got set, however, - this can cause deadlocks. In the former case, if a job is injected but no thread ever - wakes to process it, the injector will likely block forever. In the latter case, - imagine this scenario: - - thread A calls join, forking a task T1, then executing task T2 - - thread B steals T1, forks a task T3, and executes T4. - - thread A completes task T2 and blocks on T1 - - thread A steals task T3 from thread B - - thread B finishes T4 and goes to sleep, blocking on T3 - - thread A completes task T3 and makes a wakeup, but it gets lost - At this point, thread B is still asleep and will never signal T2, so thread A will itself - go to sleep. Bad. - -It turns out that guaranteeing we don't miss a wakeup while retaining -good performance is fairly tricky. This is because of some details of -the C++11 memory model. But let's ignore those for now and generally -assume sequential consistency. In that case, our scheme should work -perfectly. - -Even if you assume seqcst, though, ensuring that you don't miss -wakeups can be fairly tricky in the absence of a central queue. For -example, consider the simplest scheme: imagine we just had a boolean -flag indicating whether anyone was asleep. Then you could imagine that -when workers find no work, they flip this flag to true. When work is -published, if the flag is true, we issue a wakeup. - -The problem here is that checking for new work is not an atomic -action. So it's possible that worker 1 could start looking for work -and (say) see that worker 0's queue is empty and then search workers -2..N. While that searching is taking place, worker 0 publishes some -new work. At the time when the new work is published, the "anyone -sleeping?" flag is still false, so nothing happens. Then worker 1, who -failed to find any work, goes to sleep --- completely missing the wakeup! - -We use the "sleepy worker" idea to sidestep this problem. Under our -scheme, instead of going right to sleep at the end, worker 1 would -become sleepy. Worker 1 would then do **at least** one additional -scan. During this scan, they should find the work published by worker -0, so they will stop being sleepy and go back to work (here of course -we are assuming that no one else has stolen the worker 0 work yet; if -someone else stole it, worker 1 may still go to sleep, but that's ok, -since there is no more work to be had). - -Now you may be wondering -- how does being sleepy help? What if, -instead of publishing its job right before worker 1 became sleepy, -worker 0 wait until right before worker 1 was going to go to sleep? In -other words, the sequence was like this: - -- worker 1 gets sleepy -- worker 1 starts its scan, scanning worker 0's deque -- worker 0 publishes its job, but nobody is sleeping yet, so no wakeups occur -- worker 1 finshes its scan, goes to sleep, missing the wakeup - -The reason that this doesn't occur is because, when worker 0 publishes -its job, it will see that there is a sleepy worker. It will clear the -global state to 0. Then, when worker 1 its scan, it will notice that -it is no longer sleepy, and hence it will not go to sleep. Instead it -will awaken and keep searching for work. - -The sleepy worker phase thus also serves as a cheap way to signal that -work is around: instead of doing the whole dance of acquiring a lock -and issuing notifications, when we publish work we can just swap a -single atomic counter and let the sleepy worker notice that on their -own. - -## Beyond seq-cst - -Unfortunately, the C++11 memory model doesn't generally guarantee -seq-cst. And, somewhat annoyingly, it's not easy for the sleep module -**in isolation** to guarantee the properties the need. The key -challenge has to do with the *synchronized-with* relation. Typically, -we try to use acquire-release reasoning, and in that case the idea is -that **if** a load observes a store, it will also observe those writes -that preceded the store. But nothing says that the load **must** -observe the store -- at least not right away. - -The place that this is most relevant is the load in the `tickle()` -routine. The routine begins by reading from the global state. If it -sees anything other than 0, it then does a swap and -- if necessary -- -acquires a lock and does a notify. This load is a seq-cst load (as are -the other accesses in tickle). This ensures that it is sensible to -talk about a tickle happening *before* a worker gets sleepy and so -forth. - -It turns out that to get things right, if we use the current tickle -routine, we have to use seq-cst operations **both in the sleep module -and when publishing work**. We'll walk through two scenarios to -show what I mean. - -### Scenario 1: get-sleepy-then-get-tickled - -This scenario shows why the operations in sleep must be seq-cst. We -want to ensure that once a worker gets sleepy, any other worker that -does a tickle will observe that. In other words, we want to ensure -that the following scenario **cannot happen**: - -1. worker 1 is blocked on latch L -2. worker 1 becomes sleepy - - becoming sleepy involves a CAS on the global state to set it to 4 ("worker 1 is sleepy") -3. worker 0 sets latch L -4. worker 0 tickles **but does not see that worker 0 is sleepy** - -Let's diagram this. The notation `read_xxx(A) = V` means that a read -of location `A` was executed with the result `V`. The `xxx` is the -ordering and the location `A` is either `L` (latch) or `S` (global -state). I will leave the ordering on the latch as `xxx` as it is not -relevant here. The numbers correspond to the steps above. - -``` - worker 0 worker 1 - | +- 2: cas_sc(S, 4) -s| 3: write_xxx(L) + -b| 4: read_sc(S) = ??? <-sc-+ - v -``` - -Clearly, this cannot happen with sc orderings, because read 4 will -always return `4` here. However, if we tried to use acquire-release -orderings on the global state, then there would be **no guarantee** -that the tickle will observe that a sleepy worker occurred. We would -be guaranteed only that worker 0 would **eventually** observe that -worker 1 had become sleepy (and, at that time, that it would see other -writes). But it could take time -- and if we indeed miss that worker 1 -is sleepy, it could lead to deadlock or loss of efficiency, as -explained earlier. - -### Scenario 2: tickle-then-get-sleepy - - - -This scenario shows why latch operations must *also* be seq-cst (and, -more generally, any operations that publish work before a tickle). We -wish to ensure that this ordering of events **cannot occur**: - -1. worker 1 is blocked on latch L -2. worker 1 reads latch L, sees false, starts searching for work -3. worker 0 sets latch L -4. worker 0 tickles - - the tickle reads from the global state, sees 0 -5. worker 1 finishes searching, becomes sleepy - - becoming sleepy involves a CAS on the global state to set it to 4 ("worker 1 is sleepy") -6. worker 1 reads latch L **but does not see that worker 0 set it** -7. worker 1 may then proceed to become sleepy - -In other words, we want to ensure that if worker 0 sets a latch and -does a tickle *before worker 1 gets sleepy*, then worker 1 will -observe that latch as set when it calls probe. We'll see that, with -the current scheme, this implies that the latch memory orderings must -be seq-cst as well. - -Here is the diagram: - -``` - worker 0 worker 1 - | 2: read_xxx(L) = false -s| 3: write_xxx(L, true) -b| 4: read_sc(S) = 0 -+ - | +-sc---> 5: cas_sc(S, 4) - v 6: read_xxx(L) = ??? -``` - -The diagram shows that each thread's actions are related by -*sequenced-before* (sb). Moreover the read and write of `S` are -related by `sc` (the seq-cst ordering). However, and this is crucial, -this **does not** imply that oper 4 *synchronizes-with* oper 5. This -is because a read never synchronizes-with a store, only the -reverse. Hence, if the latch were using acq-rel orderings, it would be -legal for oper 6 to return false. But if the latch were to use an -**sc** ordering itself, then we know that oper 6 must return true, -since `3 -sc-> 4 -sc-> 5 -sc-> 6`. - -**Note** that this means that, before we tickle, we must execute some -seq-cst stores to publish our work (and during the scan we must load -from those same locations) **if we wish to guarantee that the work we -published WILL be seen by the other threads** (as opposed to -*may*). This is true for setting a latch -- if a latch is set but -another thread misses it, then the system could deadlock. However, in -the case of pushing new work to a deque, we choose not to use a seqcst -ordering. This is for several reasons: - -- If we miss a wakeup, the consequences are less dire: we simply run - less efficiently (after all, the current thread will eventually - complete its current task and pop the new task off the deque). -- It is inconvenient: The deque code is beyond our control (it lies in another package). However, - we could create a dummy `AtomicBool` for each deque and do a seqcst write to it - (with whatever value) after we push to the deque, and a seqcst load whenever - we steal from the deque. -- The cost of using a dummy variable was found to be quite high for some benchmarks: - - 8-10% overhead on nbody-parreduce - - 15% overhead on increment-all - - 40% overhead on join-recursively - -### Alternative solutions - -In both cases above, our problems arose because tickle is merely -performing a seq-cst read. If we instead had tickle perform a release -*swap*, that would be a write action of the global state. No matter -the ordering mode, all writes to the same memory location have a total -ordering, and hence we would not have to worry about others storing a -value that we fail to read (as in scenario 1). Similarly, as a release -write, a swap during tickle would synchronize-with a later cas and so -scenario 2 should be averted. So you might wonder why we don't do -that. The simple reason was that it didn't perform as well! In my -measurements, many benchmarks were unaffected by using a swap, but -some of them were hit hard: - - 8-10% overhead on nbody-parreduce - - 35% overhead on increment-all - - 245% overhead on join-recursively +sleep. The system used in this code was introduced in [Rayon RFC #5]. +There is also a [video walkthrough] available. Both of those may be +valuable resources to understanding the code, though naturally they +will also grow stale over time. The comments in this file are +extracted from the RFC and meant to be kept up to date. + +[Rayon RFC #5]: https://github.com/rayon-rs/rfcs/pull/5 +[video walkthrough]: https://youtu.be/HvmQsE5M4cY + +## The `Sleep` struct + +The `Sleep` struct is embedded into each registry. It performs several functions: + +* It tracks when workers are awake or asleep. +* It decides how long a worker should look for work before it goes to sleep, + via a callback that is invoked periodically from the worker's search loop. +* It is notified when latches are set, jobs are published, or other + events occur, and it will go and wake the appropriate threads if + they are sleeping. + +## Thread states + +An **active** thread is one that is running tasks and doing work. + +An **idle** thread is one that is in a busy loop looking for work. It will be +actively trying to steal from other threads and searching the global injection +queue for jobs. While it does this, it will periodically call back into the +sleep module with information about its progress. + +Towards the end of the idle period, idle threads also become **sleepy**. A +**sleepy** thread is an idle thread that is *about* to go to sleep after it does +one more search (or some other number, potentially). The role of going sleepy is +to prevent a possible race condition, wherein: + +* some thread A attempts to steal work from thread B, but finds that B's queue + is empty; +* thread B posts work, but finds that no workers are sleeping and hence there is + no one to wake; +* thread A decides to go to sleep, and thus never sees the work that B posted. + +This race condition can lead to suboptimal performance because we have no +workers available. + +A **sleeping** thread is one that is blocked and which must be actively awoken. +Sleeping threads will always be blocked on the mutex/condition-variable assigned +to them in the `Sleep` struct. + +## The counters + +One of the key structs in the sleep module is `AtomicCounters`, found in +`counters.rs`. It packs four counters into one atomically managed value. These +counters fall into two categories: + +* **Thread counters**, which track the number of threads in a particular state. +* **Event counters**, which track the number of events that occurred. + +### Thread counters + +There are two thread counters, one that tracks **idle** threads and one that +tracks **sleeping** threads. These are incremented and decremented as threads +enter and leave the idle/sleeping state in a fairly straightforward fashion. It +is important, however, that the thread counters are maintained atomically with +the event counters to prevent race conditions between new work being posted and +threads falling asleep. In other words, when new work is posted, it may need to +wake sleeping threads, and so we need a clear ordering whether the *work was +posted first* or the *thread fell asleep first*. + +### Event counters + +There are two event counters, the **jobs event counter** and the **sleepy +event counter**. Event counters, unlike thread counters, are never +decremented (modulo rollover, more on that later). They are simply advanced forward each time some event occurs. + +The jobs event counter is, **conceptually**, incremented each time a new job is posted. The role of this is that, when a thread becomes sleepy, it can read the jobs event counter to see how many jobs were posted up till that point. Then, when it goes to sleep, it can atomically do two things: + +* Verify that the event counter has not changed, and hence no new work was posted; +* Increment the sleepy thread counter, so that if any new work comes later, it + will know that there are sleeping threads that may have to be awoken. + +This simple picture, however, isn't quite what we do -- the problem is two-fold. +First, it would require that each time a new job is posted, we do a write +operation to a single global counter, which would be a performance bottleneck +(how much of one? I'm not sure, measurements would be good!). + +Second, it would not account for rollover -- it is possible, after all, that +enough events have been posted that the counter **has** rolled all the way back +to its original value! + +To address the first point, we keep a separate **sleepy event counter** that +(sort of) tracks how many times threads have become sleepy. Both event counters +begin at zero. Thus, after a new job is posted, we can check if they are still +equal to one another. If so, we can simply post our job, and we can now say that +the job was posted *before* any subsequent thread got sleepy -- therefore, if a +thread becomes sleepy after that, that thread will find the job when it does its +final search for work. + +If however we see that the sleepy event counter (SEC) is **not** equal to the jobs event counter (JEC), that indicates that some thread has become sleepy since work was last posted. This means we have to do more work. What we do, specifically, is to set the JEC to be equal to the SEC, and we do this atomically. + +Meanwhile, the sleepy thread, before it goes to sleep, will read the counters again and check that the JEC has not changed since it got sleepy. So, if new work *was* posted, then the JEC will have changed (to be equal to the SEC), and the sleepy thread will go back to the idle state and search again (and then possibly become sleepy again). + +This is the high-level idea, anyway. Things get subtle when you consider that there are many threads, and also that we must account for rollover. + +## Actual procedure + +Let's document the precise procedure we follow, first. In each case, all the operations happen atomically, which is easy to implement through a compare and swap. + +* To announce a thread is idle: + * IdleThreads += 1 +* To announce a thread found work: + * IdleThreads -= 1 + * If IdleThreads == SleepingThreads, return NeedToWake +* To announce a thread is sleepy: + * If SleepyEventCounter == MAX, set JobsEventCounter = SleepyEventCounter = 0 + * Else, set SleepyEventCounter = SleepyEventCounter + 1 +* To post work: + * If JobsEventCounter != SleepyEventCounter: + * JobsEventCounter = SleepyEventCounter + * NeedToWake = decide(IdleThreads, SleepingThreads) +* To fall asleep: + * If JobsEventCounter > SleepyEventCounter: + * become idle + * SleepingThreads += 1 + +### Accounting for rollover + +Let's consider three threads, A, B, and C: + +* A becomes sleepy, incrementing SleepyEventCounter to get SEC_A (which is MAX) +* In a loop: + * B gets sleepy, reseting SleepyEventCounter to 0 + * B posts work, diff --git a/rayon-core/src/sleep/counters.rs b/rayon-core/src/sleep/counters.rs index 15046b85c..5a180425e 100644 --- a/rayon-core/src/sleep/counters.rs +++ b/rayon-core/src/sleep/counters.rs @@ -5,8 +5,8 @@ pub(super) struct AtomicCounters { /// /// Consider the value `0x1111_2222_3333_4444`: /// - /// - The bits 0x1111 are the **jobs counter**. - /// - The bits 0x2222 are the **sleepy counter**. + /// - The bits 0x1111 are the **jobs event counter**. + /// - The bits 0x2222 are the **sleepy event counter**. /// - The bits 0x3333 are the number of **idle threads**. /// - The bits 0x4444 are the number of **sleeping threads**. /// @@ -20,10 +20,10 @@ pub(super) struct Counters { } #[derive(Copy, Clone, Debug, PartialEq, PartialOrd)] -pub(super) struct SleepyCounter(u16); +pub(super) struct SleepyEventCounter(u16); #[derive(Copy, Clone, Debug, PartialEq, PartialOrd)] -pub(super) struct JobsCounter(u16); +pub(super) struct JobsEventCounter(u16); const ONE_SLEEPING: u64 = 0x0000_0000_0000_0001; const ONE_IDLE: u64 = 0x0000_0000_0001_0000; @@ -35,8 +35,8 @@ const IDLE_SHIFT: u64 = 1 * 16; const SLEEPY_SHIFT: u64 = 2 * 16; const JOBS_SHIFT: u64 = 3 * 16; -pub(super) const INVALID_SLEEPY_COUNTER: SleepyCounter = SleepyCounter(std::u16::MAX); -pub(super) const ZERO_SLEEPY_COUNTER: SleepyCounter = SleepyCounter(0); +pub(super) const INVALID_SLEEPY_COUNTER: SleepyEventCounter = SleepyEventCounter(std::u16::MAX); +pub(super) const ZERO_SLEEPY_COUNTER: SleepyEventCounter = SleepyEventCounter(0); impl AtomicCounters { pub(super) fn new() -> AtomicCounters { @@ -45,6 +45,9 @@ impl AtomicCounters { } } + /// Load and return the current value of the various counters. + /// This value can then be given to other method which will + /// attempt to update the counters via compare-and-swap. pub(super) fn load(&self, ordering: Ordering) -> Counters { Counters::new(self.value.load(ordering)) } @@ -62,6 +65,9 @@ impl AtomicCounters { self.value.fetch_add(ONE_IDLE, Ordering::SeqCst); } + /// Attempt to increment the number of sleepy threads with a compare and + /// swap. Returns false if this attempt failed (in which case no changed was + /// made). #[inline] pub(super) fn try_add_sleepy_thread(&self, old_value: Counters) -> bool { debug_assert!( @@ -156,12 +162,12 @@ impl Counters { Counters { word } } - pub(super) fn jobs_counter(self) -> JobsCounter { - JobsCounter(select_u16(self.word, JOBS_SHIFT)) + pub(super) fn jobs_counter(self) -> JobsEventCounter { + JobsEventCounter(select_u16(self.word, JOBS_SHIFT)) } - pub(super) fn sleepy_counter(self) -> SleepyCounter { - SleepyCounter(select_u16(self.word, SLEEPY_SHIFT)) + pub(super) fn sleepy_counter(self) -> SleepyEventCounter { + SleepyEventCounter(select_u16(self.word, SLEEPY_SHIFT)) } pub(super) fn raw_idle_threads(self) -> u16 { @@ -177,7 +183,7 @@ impl Counters { } } -impl SleepyCounter { +impl SleepyEventCounter { pub(super) fn is_max(self) -> bool { self.0 == std::u16::MAX } @@ -187,26 +193,26 @@ impl SleepyCounter { } } -impl PartialOrd for SleepyCounter { - fn partial_cmp(&self, other: &JobsCounter) -> Option<::std::cmp::Ordering> { +impl PartialOrd for SleepyEventCounter { + fn partial_cmp(&self, other: &JobsEventCounter) -> Option<::std::cmp::Ordering> { PartialOrd::partial_cmp(&self.0, &other.0) } } -impl PartialOrd for JobsCounter { - fn partial_cmp(&self, other: &SleepyCounter) -> Option<::std::cmp::Ordering> { +impl PartialOrd for JobsEventCounter { + fn partial_cmp(&self, other: &SleepyEventCounter) -> Option<::std::cmp::Ordering> { PartialOrd::partial_cmp(&self.0, &other.0) } } -impl PartialEq for SleepyCounter { - fn eq(&self, other: &JobsCounter) -> bool { +impl PartialEq for SleepyEventCounter { + fn eq(&self, other: &JobsEventCounter) -> bool { PartialEq::eq(&self.0, &other.0) } } -impl PartialEq for JobsCounter { - fn eq(&self, other: &SleepyCounter) -> bool { +impl PartialEq for JobsEventCounter { + fn eq(&self, other: &SleepyEventCounter) -> bool { PartialEq::eq(&self.0, &other.0) } } diff --git a/rayon-core/src/sleep/mod.rs b/rayon-core/src/sleep/mod.rs index 7202608d9..ae3c133ff 100644 --- a/rayon-core/src/sleep/mod.rs +++ b/rayon-core/src/sleep/mod.rs @@ -12,9 +12,16 @@ use std::usize; mod counters; use self::counters::{ - AtomicCounters, Counters, SleepyCounter, INVALID_SLEEPY_COUNTER, ZERO_SLEEPY_COUNTER, + AtomicCounters, Counters, SleepyEventCounter, INVALID_SLEEPY_COUNTER, ZERO_SLEEPY_COUNTER, }; +/// The `Sleep` struct is embedded into each registry. It governs the waking and sleeping +/// of workers. It has callbacks that are invoked periodically at significant events, +/// such as when workers are looping and looking for work, when latches are set, or when +/// jobs are published, and it either blocks threads or wakes them in response to these +/// events. See the [`README.md`] in this module for more details. +/// +/// [`README.md`] README.md pub(super) struct Sleep { logger: Logger, @@ -39,7 +46,7 @@ pub(super) struct IdleState { /// Once we become sleepy, what was the sleepy counter value? /// Set to `INVALID_SLEEPY_COUNTER` otherwise. - sleepy_counter: SleepyCounter, + sleepy_counter: SleepyEventCounter, } /// The "sleep state" for an individual worker. @@ -112,7 +119,7 @@ impl Sleep { } #[cold] - fn announce_sleepy(&self, worker_index: usize) -> SleepyCounter { + fn announce_sleepy(&self, worker_index: usize) -> SleepyEventCounter { loop { let counters = self.counters.load(Ordering::Relaxed); let sleepy_counter = counters.sleepy_counter(); From d5113642b496fd4f91a3fa2a89e8157a1dbe3721 Mon Sep 17 00:00:00 2001 From: Niko Matsakis Date: Sat, 11 Apr 2020 18:26:50 +0000 Subject: [PATCH 17/33] adopt the "ignore rollect, even-odd JEC" design --- rayon-core/src/log.rs | 2 +- rayon-core/src/sleep/counters.rs | 116 +++++++++---------------------- rayon-core/src/sleep/mod.rs | 83 +++++++--------------- 3 files changed, 59 insertions(+), 142 deletions(-) diff --git a/rayon-core/src/log.rs b/rayon-core/src/log.rs index cf073f355..99695d1a0 100644 --- a/rayon-core/src/log.rs +++ b/rayon-core/src/log.rs @@ -56,7 +56,7 @@ pub(super) enum Event { /// Indicates that an idle worker is getting sleepy. `sleepy_counter` is the internal /// sleep state that we saw at the time. - ThreadSleepy { worker: usize, sleepy_counter: u16 }, + ThreadSleepy { worker: usize, jobs_counter: u16 }, /// Indicates that the thread's attempt to fall asleep was /// interrupted because the latch was set. (This is not, in and of diff --git a/rayon-core/src/sleep/counters.rs b/rayon-core/src/sleep/counters.rs index 5a180425e..5772fa6f4 100644 --- a/rayon-core/src/sleep/counters.rs +++ b/rayon-core/src/sleep/counters.rs @@ -19,24 +19,35 @@ pub(super) struct Counters { word: u64, } -#[derive(Copy, Clone, Debug, PartialEq, PartialOrd)] -pub(super) struct SleepyEventCounter(u16); - +/// A value read from the **Jobs Event Counter**. +/// See the [`README.md`](README.md) for more +/// coverage of how the jobs event counter works. #[derive(Copy, Clone, Debug, PartialEq, PartialOrd)] pub(super) struct JobsEventCounter(u16); +impl JobsEventCounter { + pub(super) const INVALID: JobsEventCounter = JobsEventCounter(JOBS_MAX); + + fn as_usize(self) -> usize { + self.0 as usize + } + + /// Returns true if there were sleepy workers pending when this reading was + /// taken. This is determined by checking whether the value is odd (sleepy + /// workers) or even (work published since last sleepy worker got sleepy, + /// though they may not have seen it yet). + pub(super) fn is_sleepy(self) -> bool { + (self.as_usize() & 1) != 0 + } +} + const ONE_SLEEPING: u64 = 0x0000_0000_0000_0001; const ONE_IDLE: u64 = 0x0000_0000_0001_0000; -const ONE_SLEEPY: u64 = 0x0000_0001_0000_0000; -const SLEEPY_ROLLVER_MASK: u64 = 0x0000_0000_FFFF_FFFF; -const NO_JOBS_MASK: u64 = 0x0000_FFFF_FFFF_FFFF; +const ONE_JEC: u64 = 0x0000_0001_0000_0000; const SLEEPING_SHIFT: u64 = 0; const IDLE_SHIFT: u64 = 1 * 16; -const SLEEPY_SHIFT: u64 = 2 * 16; -const JOBS_SHIFT: u64 = 3 * 16; - -pub(super) const INVALID_SLEEPY_COUNTER: SleepyEventCounter = SleepyEventCounter(std::u16::MAX); -pub(super) const ZERO_SLEEPY_COUNTER: SleepyEventCounter = SleepyEventCounter(0); +const JOBS_SHIFT: u64 = 2 * 16; +const JOBS_MAX: u16 = 0xFFFF; impl AtomicCounters { pub(super) fn new() -> AtomicCounters { @@ -65,23 +76,20 @@ impl AtomicCounters { self.value.fetch_add(ONE_IDLE, Ordering::SeqCst); } - /// Attempt to increment the number of sleepy threads with a compare and - /// swap. Returns false if this attempt failed (in which case no changed was - /// made). - #[inline] - pub(super) fn try_add_sleepy_thread(&self, old_value: Counters) -> bool { - debug_assert!( - !old_value.sleepy_counter().is_max(), - "try_add_sleepy_thread: old_value {:?} has max sleepy threads", - old_value, - ); - let new_value = Counters::new(old_value.word + ONE_SLEEPY); + /// Attempts to increment the jobs event counter by one, returning true + /// if it succeeded. This can be used for two purposes: + /// + /// * If a thread is getting sleepy, and the JEC is even, then it will attempt + /// to increment to an odd value. + /// * If a thread is publishing work, and the JEC is odd, then it will attempt + /// to increment to an event value. + pub(super) fn try_increment_jobs_event_counter(&self, old_value: Counters) -> bool { + let new_value = Counters::new(old_value.word + ONE_JEC); self.try_exchange(old_value, new_value, Ordering::SeqCst) } /// Subtracts an idle thread. This cannot fail; it returns the /// number of sleeping threads to wake up (if any). - #[inline] pub(super) fn sub_idle_thread(&self) -> u16 { let old_value = Counters::new(self.value.fetch_sub(ONE_IDLE, Ordering::SeqCst)); debug_assert!( @@ -128,29 +136,6 @@ impl AtomicCounters { self.try_exchange(old_value, new_value, Ordering::SeqCst) } - - #[inline] - pub(super) fn try_replicate_sleepy_counter(&self, old_value: Counters) -> bool { - let sc = old_value.sleepy_counter(); - - // clear jobs counter - let word_without_jc = old_value.word & NO_JOBS_MASK; - - // replace with sleepy counter - let sc_shifted_to_jc = (sc.0 as u64) << JOBS_SHIFT; - let new_value = Counters::new(word_without_jc | sc_shifted_to_jc); - - debug_assert!(new_value.sleepy_counter() == old_value.sleepy_counter()); - debug_assert!(new_value.jobs_counter() == old_value.sleepy_counter()); - - self.try_exchange(old_value, new_value, Ordering::SeqCst) - } - - #[inline] - pub(super) fn try_rollover_jobs_and_sleepy_counters(&self, old_value: Counters) -> bool { - let new_value = Counters::new(old_value.word & SLEEPY_ROLLVER_MASK); - self.try_exchange(old_value, new_value, Ordering::SeqCst) - } } fn select_u16(word: u64, shift: u64) -> u16 { @@ -166,10 +151,6 @@ impl Counters { JobsEventCounter(select_u16(self.word, JOBS_SHIFT)) } - pub(super) fn sleepy_counter(self) -> SleepyEventCounter { - SleepyEventCounter(select_u16(self.word, SLEEPY_SHIFT)) - } - pub(super) fn raw_idle_threads(self) -> u16 { select_u16(self.word, IDLE_SHIFT) } @@ -183,47 +164,12 @@ impl Counters { } } -impl SleepyEventCounter { - pub(super) fn is_max(self) -> bool { - self.0 == std::u16::MAX - } - - pub(super) fn as_u16(self) -> u16 { - self.0 - } -} - -impl PartialOrd for SleepyEventCounter { - fn partial_cmp(&self, other: &JobsEventCounter) -> Option<::std::cmp::Ordering> { - PartialOrd::partial_cmp(&self.0, &other.0) - } -} - -impl PartialOrd for JobsEventCounter { - fn partial_cmp(&self, other: &SleepyEventCounter) -> Option<::std::cmp::Ordering> { - PartialOrd::partial_cmp(&self.0, &other.0) - } -} - -impl PartialEq for SleepyEventCounter { - fn eq(&self, other: &JobsEventCounter) -> bool { - PartialEq::eq(&self.0, &other.0) - } -} - -impl PartialEq for JobsEventCounter { - fn eq(&self, other: &SleepyEventCounter) -> bool { - PartialEq::eq(&self.0, &other.0) - } -} - impl std::fmt::Debug for Counters { fn fmt(&self, fmt: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { let word = format!("{:016x}", self.word); fmt.debug_struct("Counters") .field("word", &word) .field("jobs", &self.jobs_counter().0) - .field("sleepy", &self.sleepy_counter()) .field("idle", &self.raw_idle_threads()) .field("sleeping", &self.sleeping_threads()) .finish() diff --git a/rayon-core/src/sleep/mod.rs b/rayon-core/src/sleep/mod.rs index ae3c133ff..b9473bf84 100644 --- a/rayon-core/src/sleep/mod.rs +++ b/rayon-core/src/sleep/mod.rs @@ -11,9 +11,7 @@ use std::thread; use std::usize; mod counters; -use self::counters::{ - AtomicCounters, Counters, SleepyEventCounter, INVALID_SLEEPY_COUNTER, ZERO_SLEEPY_COUNTER, -}; +use self::counters::{AtomicCounters, JobsEventCounter}; /// The `Sleep` struct is embedded into each registry. It governs the waking and sleeping /// of workers. It has callbacks that are invoked periodically at significant events, @@ -46,7 +44,7 @@ pub(super) struct IdleState { /// Once we become sleepy, what was the sleepy counter value? /// Set to `INVALID_SLEEPY_COUNTER` otherwise. - sleepy_counter: SleepyEventCounter, + jobs_counter: JobsEventCounter, } /// The "sleep state" for an individual worker. @@ -83,7 +81,7 @@ impl Sleep { IdleState { worker_index, rounds: 0, - sleepy_counter: INVALID_SLEEPY_COUNTER, + jobs_counter: JobsEventCounter::INVALID, } } @@ -106,7 +104,7 @@ impl Sleep { thread::yield_now(); idle_state.rounds += 1; } else if idle_state.rounds == ROUNDS_UNTIL_SLEEPY { - idle_state.sleepy_counter = self.announce_sleepy(idle_state.worker_index); + idle_state.jobs_counter = self.announce_sleepy(idle_state.worker_index); idle_state.rounds += 1; thread::yield_now(); } else if idle_state.rounds < ROUNDS_UNTIL_SLEEPING { @@ -119,29 +117,17 @@ impl Sleep { } #[cold] - fn announce_sleepy(&self, worker_index: usize) -> SleepyEventCounter { + fn announce_sleepy(&self, worker_index: usize) -> JobsEventCounter { loop { let counters = self.counters.load(Ordering::Relaxed); - let sleepy_counter = counters.sleepy_counter(); - if sleepy_counter.is_max() { - if self - .counters - .try_rollover_jobs_and_sleepy_counters(counters) - { - self.logger.log(|| ThreadSleepy { - worker: worker_index, - sleepy_counter: 0, - }); - return ZERO_SLEEPY_COUNTER; - } - } else { - if self.counters.try_add_sleepy_thread(counters) { - self.logger.log(|| ThreadSleepy { - worker: worker_index, - sleepy_counter: sleepy_counter.as_u16(), - }); - return sleepy_counter; - } + let jobs_counter = counters.jobs_counter(); + if jobs_counter.is_sleepy() || self.counters.try_increment_jobs_event_counter(counters) + { + self.logger.log(|| ThreadSleepy { + worker: worker_index, + jobs_counter: 0, + }); + return jobs_counter; } } } @@ -177,7 +163,7 @@ impl Sleep { loop { let counters = self.counters.load(Ordering::SeqCst); - if counters.jobs_counter() > idle_state.sleepy_counter { + if counters.jobs_counter() != idle_state.jobs_counter { self.logger.log(|| ThreadSleepInterruptedByJob { worker: worker_index, }); @@ -283,11 +269,18 @@ impl Sleep { /// Common helper for `new_injected_jobs` and `new_internal_jobs`. #[inline] fn new_jobs(&self, source_worker_index: usize, num_jobs: u32, queue_was_empty: bool) { - let mut counters = self.counters.load(Ordering::SeqCst); + let mut counters; - // If we find that the jobs counter is out of date, we have to fix that. - if counters.jobs_counter() != counters.sleepy_counter() { - self.sync_jobs_counter(&mut counters); + // Read the counters and -- if sleepy workers have announced themselves + // -- announce that there is now work available. The final value of `counters` + // with which we exit the loop thus corresponds to a state when + loop { + counters = self.counters.load(Ordering::SeqCst); + if !counters.jobs_counter().is_sleepy() { + break; + } else if self.counters.try_increment_jobs_event_counter(counters) { + break; + } } let num_awake_but_idle = counters.awake_but_idle_threads(); @@ -321,28 +314,6 @@ impl Sleep { } } - /// Invoked when we find that the "jobs counter" is not equal to - /// the "sleepy counter". This means that there may be threads - /// actively going to sleep. In that case, we have to synchronize - /// the jobs counter with the sleepy counter using a - /// compare-exchange. If it happens that this fails, then the - /// sleepy thread may have actually gone to sleep, so we re-load - /// the counters word. - #[cold] - fn sync_jobs_counter(&self, counters: &mut Counters) { - loop { - if counters.jobs_counter() == counters.sleepy_counter() { - return; - } - - if self.counters.try_replicate_sleepy_counter(*counters) { - return; - } - - *counters = self.counters.load(Ordering::SeqCst); - } - } - #[cold] fn wake_any_threads(&self, mut num_to_wake: u32) { if num_to_wake > 0 { @@ -388,11 +359,11 @@ impl Sleep { impl IdleState { fn wake_fully(&mut self) { self.rounds = 0; - self.sleepy_counter = INVALID_SLEEPY_COUNTER; + self.jobs_counter = JobsEventCounter::INVALID; } fn wake_partly(&mut self) { self.rounds = ROUNDS_UNTIL_SLEEPY; - self.sleepy_counter = INVALID_SLEEPY_COUNTER; + self.jobs_counter = JobsEventCounter::INVALID; } } From e460cc41a602e5a9a9ca6e9580c2a09f6a400e61 Mon Sep 17 00:00:00 2001 From: Niko Matsakis Date: Sat, 11 Apr 2020 18:36:13 +0000 Subject: [PATCH 18/33] use usize instead of u16 --- rayon-core/src/sleep/counters.rs | 29 ++++++++++++++++------------- 1 file changed, 16 insertions(+), 13 deletions(-) diff --git a/rayon-core/src/sleep/counters.rs b/rayon-core/src/sleep/counters.rs index 5772fa6f4..d0bfe4221 100644 --- a/rayon-core/src/sleep/counters.rs +++ b/rayon-core/src/sleep/counters.rs @@ -23,13 +23,13 @@ pub(super) struct Counters { /// See the [`README.md`](README.md) for more /// coverage of how the jobs event counter works. #[derive(Copy, Clone, Debug, PartialEq, PartialOrd)] -pub(super) struct JobsEventCounter(u16); +pub(super) struct JobsEventCounter(usize); impl JobsEventCounter { pub(super) const INVALID: JobsEventCounter = JobsEventCounter(JOBS_MAX); fn as_usize(self) -> usize { - self.0 as usize + self.0 } /// Returns true if there were sleepy workers pending when this reading was @@ -47,7 +47,10 @@ const ONE_JEC: u64 = 0x0000_0001_0000_0000; const SLEEPING_SHIFT: u64 = 0; const IDLE_SHIFT: u64 = 1 * 16; const JOBS_SHIFT: u64 = 2 * 16; -const JOBS_MAX: u16 = 0xFFFF; +const JOBS_MAX: usize = 0xFFFF; +const THREADS_MAX: usize = 0xFFFF; +const THREADS_BITS: usize = 0xFFFF; +const JOBS_BITS: usize = std::usize::MAX; impl AtomicCounters { pub(super) fn new() -> AtomicCounters { @@ -90,7 +93,7 @@ impl AtomicCounters { /// Subtracts an idle thread. This cannot fail; it returns the /// number of sleeping threads to wake up (if any). - pub(super) fn sub_idle_thread(&self) -> u16 { + pub(super) fn sub_idle_thread(&self) -> usize { let old_value = Counters::new(self.value.fetch_sub(ONE_IDLE, Ordering::SeqCst)); debug_assert!( old_value.raw_idle_threads() > 0, @@ -126,7 +129,7 @@ impl AtomicCounters { old_value, ); debug_assert!( - old_value.sleeping_threads() < std::u16::MAX, + old_value.sleeping_threads() < THREADS_MAX, "try_add_sleeping_thread: old_value {:?} has too many sleeping threads", old_value, ); @@ -138,8 +141,8 @@ impl AtomicCounters { } } -fn select_u16(word: u64, shift: u64) -> u16 { - (word >> shift) as u16 +fn select_bits(word: u64, shift: u64, bits: usize) -> usize { + ((word >> shift) as usize) & bits } impl Counters { @@ -148,19 +151,19 @@ impl Counters { } pub(super) fn jobs_counter(self) -> JobsEventCounter { - JobsEventCounter(select_u16(self.word, JOBS_SHIFT)) + JobsEventCounter(select_bits(self.word, JOBS_SHIFT, JOBS_BITS)) } - pub(super) fn raw_idle_threads(self) -> u16 { - select_u16(self.word, IDLE_SHIFT) + pub(super) fn raw_idle_threads(self) -> usize { + select_bits(self.word, IDLE_SHIFT, THREADS_BITS) } - pub(super) fn awake_but_idle_threads(self) -> u16 { + pub(super) fn awake_but_idle_threads(self) -> usize { self.raw_idle_threads() - self.sleeping_threads() } - pub(super) fn sleeping_threads(self) -> u16 { - select_u16(self.word, SLEEPING_SHIFT) + pub(super) fn sleeping_threads(self) -> usize { + select_bits(self.word, SLEEPING_SHIFT, THREADS_BITS) } } From 8a99d668c4de6870f09231ef6c738666708b8daf Mon Sep 17 00:00:00 2001 From: Niko Matsakis Date: Sat, 11 Apr 2020 18:50:19 +0000 Subject: [PATCH 19/33] rename JOBS to JEC, comment constants --- rayon-core/src/sleep/counters.rs | 34 +++++++++++++++++++++++++++----- 1 file changed, 29 insertions(+), 5 deletions(-) diff --git a/rayon-core/src/sleep/counters.rs b/rayon-core/src/sleep/counters.rs index d0bfe4221..9aadb42a5 100644 --- a/rayon-core/src/sleep/counters.rs +++ b/rayon-core/src/sleep/counters.rs @@ -26,7 +26,7 @@ pub(super) struct Counters { pub(super) struct JobsEventCounter(usize); impl JobsEventCounter { - pub(super) const INVALID: JobsEventCounter = JobsEventCounter(JOBS_MAX); + pub(super) const INVALID: JobsEventCounter = JobsEventCounter(JEC_MAX); fn as_usize(self) -> usize { self.0 @@ -41,16 +41,40 @@ impl JobsEventCounter { } } +/// Constant that can be added to add one sleeping thread. const ONE_SLEEPING: u64 = 0x0000_0000_0000_0001; + +/// Constant that can be added to add one idle thread. const ONE_IDLE: u64 = 0x0000_0000_0001_0000; + +/// Constant that can be added to add one to the JEC. const ONE_JEC: u64 = 0x0000_0001_0000_0000; + +/// Bits to shift to select the sleeping threads +/// (used with `select_bits`). const SLEEPING_SHIFT: u64 = 0; + +/// Bits to shift to select the idle threads +/// (used with `select_bits`). const IDLE_SHIFT: u64 = 1 * 16; -const JOBS_SHIFT: u64 = 2 * 16; -const JOBS_MAX: usize = 0xFFFF; + +/// Bits to shift to select the JEC +/// (use JOBS_BITS). +const JEC_SHIFT: u64 = 2 * 16; + +/// Max value for the jobs event counter. +const JEC_MAX: usize = 0xFFFF; + +/// Max value for the thread counters. const THREADS_MAX: usize = 0xFFFF; + +/// Mask to select the sleeping|idle thread value, after +/// shifting (used with `select_bits`) const THREADS_BITS: usize = 0xFFFF; -const JOBS_BITS: usize = std::usize::MAX; + +/// Mask to select the JEC, after +/// shifting (used with `select_bits`) +const JEC_BITS: usize = std::usize::MAX; impl AtomicCounters { pub(super) fn new() -> AtomicCounters { @@ -151,7 +175,7 @@ impl Counters { } pub(super) fn jobs_counter(self) -> JobsEventCounter { - JobsEventCounter(select_bits(self.word, JOBS_SHIFT, JOBS_BITS)) + JobsEventCounter(select_bits(self.word, JEC_SHIFT, JEC_BITS)) } pub(super) fn raw_idle_threads(self) -> usize { From 2c22c804ff200382f0af88e8681845b4eebb4ae7 Mon Sep 17 00:00:00 2001 From: Niko Matsakis Date: Sat, 11 Apr 2020 18:52:32 +0000 Subject: [PATCH 20/33] rename from INVALID to MAX (it is, after all, valid) --- rayon-core/src/sleep/counters.rs | 2 +- rayon-core/src/sleep/mod.rs | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/rayon-core/src/sleep/counters.rs b/rayon-core/src/sleep/counters.rs index 9aadb42a5..f1867e7f9 100644 --- a/rayon-core/src/sleep/counters.rs +++ b/rayon-core/src/sleep/counters.rs @@ -26,7 +26,7 @@ pub(super) struct Counters { pub(super) struct JobsEventCounter(usize); impl JobsEventCounter { - pub(super) const INVALID: JobsEventCounter = JobsEventCounter(JEC_MAX); + pub(super) const MAX: JobsEventCounter = JobsEventCounter(JEC_MAX); fn as_usize(self) -> usize { self.0 diff --git a/rayon-core/src/sleep/mod.rs b/rayon-core/src/sleep/mod.rs index b9473bf84..37c7407eb 100644 --- a/rayon-core/src/sleep/mod.rs +++ b/rayon-core/src/sleep/mod.rs @@ -81,7 +81,7 @@ impl Sleep { IdleState { worker_index, rounds: 0, - jobs_counter: JobsEventCounter::INVALID, + jobs_counter: JobsEventCounter::MAX, } } @@ -359,11 +359,11 @@ impl Sleep { impl IdleState { fn wake_fully(&mut self) { self.rounds = 0; - self.jobs_counter = JobsEventCounter::INVALID; + self.jobs_counter = JobsEventCounter::MAX; } fn wake_partly(&mut self) { self.rounds = ROUNDS_UNTIL_SLEEPY; - self.jobs_counter = JobsEventCounter::INVALID; + self.jobs_counter = JobsEventCounter::MAX; } } From 908e1371598342e7ff2fd3de03b88b7a75955b53 Mon Sep 17 00:00:00 2001 From: Niko Matsakis Date: Sat, 11 Apr 2020 18:58:05 +0000 Subject: [PATCH 21/33] WIP: rework to permit rollover at arbitrary points --- rayon-core/src/sleep/counters.rs | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/rayon-core/src/sleep/counters.rs b/rayon-core/src/sleep/counters.rs index f1867e7f9..b2dfbb8a6 100644 --- a/rayon-core/src/sleep/counters.rs +++ b/rayon-core/src/sleep/counters.rs @@ -50,6 +50,9 @@ const ONE_IDLE: u64 = 0x0000_0000_0001_0000; /// Constant that can be added to add one to the JEC. const ONE_JEC: u64 = 0x0000_0001_0000_0000; +/// Mask to zero out the JEC (but retain everything else). +const ZERO_JEC_MASK: u64 = 0x0000_0000_FFFF_FFFF; + /// Bits to shift to select the sleeping threads /// (used with `select_bits`). const SLEEPING_SHIFT: u64 = 0; @@ -111,7 +114,11 @@ impl AtomicCounters { /// * If a thread is publishing work, and the JEC is odd, then it will attempt /// to increment to an event value. pub(super) fn try_increment_jobs_event_counter(&self, old_value: Counters) -> bool { - let new_value = Counters::new(old_value.word + ONE_JEC); + let new_value = if old_value.jobs_counter() == JobsEventCounter::MAX { + Counters::new(old_value.word & ZERO_JEC_MASK) + } else { + Counters::new(old_value.word + ONE_JEC) + }; self.try_exchange(old_value, new_value, Ordering::SeqCst) } From 1222cbeb1f7cad8a9984a26a5b00d8d9022a67a3 Mon Sep 17 00:00:00 2001 From: Niko Matsakis Date: Mon, 13 Apr 2020 10:09:58 +0000 Subject: [PATCH 22/33] check one last time for injected jobs before sleep --- rayon-core/src/registry.rs | 8 +++++- rayon-core/src/sleep/counters.rs | 25 +++++++++++++++-- rayon-core/src/sleep/mod.rs | 48 ++++++++++++++++++++++++-------- 3 files changed, 65 insertions(+), 16 deletions(-) diff --git a/rayon-core/src/registry.rs b/rayon-core/src/registry.rs index 40c752369..0a1a8b2d0 100644 --- a/rayon-core/src/registry.rs +++ b/rayon-core/src/registry.rs @@ -408,6 +408,10 @@ impl Registry { .new_injected_jobs(usize::MAX, injected_jobs.len() as u32, queue_was_empty); } + fn has_injected_job(&self) -> bool { + !self.injected_jobs.is_empty() + } + fn pop_injected_job(&self, worker_index: usize) -> Option { let job = self.injected_jobs.pop().ok(); if job.is_some() { @@ -721,7 +725,9 @@ impl WorkerThread { self.execute(job); idle_state = self.registry.sleep.start_looking(self.index, latch); } else { - self.registry.sleep.no_work_found(&mut idle_state, latch); + self.registry + .sleep + .no_work_found(&mut idle_state, latch, || self.registry.has_injected_job()) } } diff --git a/rayon-core/src/sleep/counters.rs b/rayon-core/src/sleep/counters.rs index b2dfbb8a6..57598bfcb 100644 --- a/rayon-core/src/sleep/counters.rs +++ b/rayon-core/src/sleep/counters.rs @@ -3,10 +3,9 @@ use std::sync::atomic::{AtomicU64, Ordering}; pub(super) struct AtomicCounters { /// Packs together a number of counters. /// - /// Consider the value `0x1111_2222_3333_4444`: + /// Consider the value `0x11112222_3333_4444`: /// - /// - The bits 0x1111 are the **jobs event counter**. - /// - The bits 0x2222 are the **sleepy event counter**. + /// - The bits 0x11112222 are the **jobs event counter**. /// - The bits 0x3333 are the number of **idle threads**. /// - The bits 0x4444 are the number of **sleeping threads**. /// @@ -131,6 +130,13 @@ impl AtomicCounters { "sub_idle_thread: old_value {:?} has no idle threads", old_value, ); + debug_assert!( + old_value.sleeping_threads() <= old_value.raw_idle_threads(), + "sub_sleeping_thread: old_value {:?} had {} sleeping threads and {} raw idle threads", + old_value, + old_value.sleeping_threads(), + old_value.raw_idle_threads(), + ); // Current heuristic: whenever an idle thread goes away, if // there are any sleeping threads, wake 'em up. @@ -150,6 +156,13 @@ impl AtomicCounters { "sub_sleeping_thread: old_value {:?} had no sleeping threads", old_value, ); + debug_assert!( + old_value.sleeping_threads() <= old_value.raw_idle_threads(), + "sub_sleeping_thread: old_value {:?} had {} sleeping threads and {} raw idle threads", + old_value, + old_value.sleeping_threads(), + old_value.raw_idle_threads(), + ); } #[inline] @@ -190,6 +203,12 @@ impl Counters { } pub(super) fn awake_but_idle_threads(self) -> usize { + debug_assert!( + self.sleeping_threads() <= self.raw_idle_threads(), + "sleeping threads: {} > raw idle threads {}", + self.sleeping_threads(), + self.raw_idle_threads() + ); self.raw_idle_threads() - self.sleeping_threads() } diff --git a/rayon-core/src/sleep/mod.rs b/rayon-core/src/sleep/mod.rs index 37c7407eb..2a40befd5 100644 --- a/rayon-core/src/sleep/mod.rs +++ b/rayon-core/src/sleep/mod.rs @@ -99,7 +99,12 @@ impl Sleep { } #[inline] - pub(super) fn no_work_found(&self, idle_state: &mut IdleState, latch: &CoreLatch) { + pub(super) fn no_work_found( + &self, + idle_state: &mut IdleState, + latch: &CoreLatch, + has_injected_jobs: impl FnOnce() -> bool, + ) { if idle_state.rounds < ROUNDS_UNTIL_SLEEPY { thread::yield_now(); idle_state.rounds += 1; @@ -112,7 +117,7 @@ impl Sleep { thread::yield_now(); } else { debug_assert_eq!(idle_state.rounds, ROUNDS_UNTIL_SLEEPING); - self.sleep(idle_state, latch); + self.sleep(idle_state, latch, has_injected_jobs); } } @@ -133,7 +138,12 @@ impl Sleep { } #[cold] - fn sleep(&self, idle_state: &mut IdleState, latch: &CoreLatch) { + fn sleep( + &self, + idle_state: &mut IdleState, + latch: &CoreLatch, + has_injected_jobs: impl FnOnce() -> bool, + ) { let worker_index = idle_state.worker_index; if !latch.get_sleepy() { @@ -189,16 +199,30 @@ impl Sleep { latch_addr: latch.addr(), }); - // Flag ourselves as asleep and wait till we are notified. + // We have one last check for injected jobs to do. This protects against + // deadlock in the very unlikely event that // - // (Note that `is_blocked` is held under a mutex and the mutex - // was acquired *before* we incremented the "sleepy - // counter". This means that whomever is coming to wake us - // will have to wait until we release the mutex in the call to - // `wait`, so they will see this boolean as true.) - *is_blocked = true; - while *is_blocked { - is_blocked = sleep_state.condvar.wait(is_blocked).unwrap(); + // - an external job is being injected while we are sleepy + // - that job triggers the rollover over the JEC such that we don't see it + // - we are the last active worker thread + if has_injected_jobs() { + // If we see an externally injected job, then we have to 'wake + // ourselves up'. (Ordinarily, `sub_sleeping_thread` is invoked by + // the one that wakes us.) + self.counters.sub_sleeping_thread(); + } else { + // If we don't see an injected job (the normal case), then flag + // ourselves as asleep and wait till we are notified. + // + // (Note that `is_blocked` is held under a mutex and the mutex was + // acquired *before* we incremented the "sleepy counter". This means + // that whomever is coming to wake us will have to wait until we + // release the mutex in the call to `wait`, so they will see this + // boolean as true.) + *is_blocked = true; + while *is_blocked { + is_blocked = sleep_state.condvar.wait(is_blocked).unwrap(); + } } // Update other state: From 2124e8331b1a7cf3a94b5738f60eca1cdc29a121 Mon Sep 17 00:00:00 2001 From: Niko Matsakis Date: Mon, 13 Apr 2020 10:20:49 +0000 Subject: [PATCH 23/33] rename "raw idle" to "inactive" --- rayon-core/src/sleep/counters.rs | 73 +++++++++++++++++++------------- rayon-core/src/sleep/mod.rs | 4 +- 2 files changed, 45 insertions(+), 32 deletions(-) diff --git a/rayon-core/src/sleep/counters.rs b/rayon-core/src/sleep/counters.rs index 57598bfcb..031b33248 100644 --- a/rayon-core/src/sleep/counters.rs +++ b/rayon-core/src/sleep/counters.rs @@ -6,7 +6,7 @@ pub(super) struct AtomicCounters { /// Consider the value `0x11112222_3333_4444`: /// /// - The bits 0x11112222 are the **jobs event counter**. - /// - The bits 0x3333 are the number of **idle threads**. + /// - The bits 0x3333 are the number of **inactive threads**. /// - The bits 0x4444 are the number of **sleeping threads**. /// /// See the struct `Counters` below. @@ -43,8 +43,9 @@ impl JobsEventCounter { /// Constant that can be added to add one sleeping thread. const ONE_SLEEPING: u64 = 0x0000_0000_0000_0001; -/// Constant that can be added to add one idle thread. -const ONE_IDLE: u64 = 0x0000_0000_0001_0000; +/// Constant that can be added to add one inactive thread. +/// An inactive thread is either idle, sleepy, or sleeping. +const ONE_INACTIVE: u64 = 0x0000_0000_0001_0000; /// Constant that can be added to add one to the JEC. const ONE_JEC: u64 = 0x0000_0001_0000_0000; @@ -56,9 +57,9 @@ const ZERO_JEC_MASK: u64 = 0x0000_0000_FFFF_FFFF; /// (used with `select_bits`). const SLEEPING_SHIFT: u64 = 0; -/// Bits to shift to select the idle threads +/// Bits to shift to select the inactive threads /// (used with `select_bits`). -const IDLE_SHIFT: u64 = 1 * 16; +const INACTIVE_SHIFT: u64 = 1 * 16; /// Bits to shift to select the JEC /// (use JOBS_BITS). @@ -70,8 +71,9 @@ const JEC_MAX: usize = 0xFFFF; /// Max value for the thread counters. const THREADS_MAX: usize = 0xFFFF; -/// Mask to select the sleeping|idle thread value, after -/// shifting (used with `select_bits`) +/// Mask to select the value of a thread counter +/// (sleepy, inactive), after shifting (used with +/// `select_bits`) const THREADS_BITS: usize = 0xFFFF; /// Mask to select the JEC, after @@ -99,10 +101,16 @@ impl AtomicCounters { .is_ok() } - /// Adds an idle thread. This cannot fail. + /// Adds an inactive thread. This cannot fail. + /// + /// This should be invoked when a thread enters its idle loop looking + /// for work. It is decremented when work is found. Note that it is + /// not decremented if the thread transitions from idle to sleepy or sleeping; + /// so the number of inactive threads is always greater-than-or-equal + /// to the number of sleeping threads. #[inline] - pub(super) fn add_idle_thread(&self) { - self.value.fetch_add(ONE_IDLE, Ordering::SeqCst); + pub(super) fn add_inactive_thread(&self) { + self.value.fetch_add(ONE_INACTIVE, Ordering::SeqCst); } /// Attempts to increment the jobs event counter by one, returning true @@ -121,24 +129,27 @@ impl AtomicCounters { self.try_exchange(old_value, new_value, Ordering::SeqCst) } - /// Subtracts an idle thread. This cannot fail; it returns the + /// Subtracts an inactive thread. This cannot fail. It is invoked + /// when a thread finds work and hence becomes active. It returns the /// number of sleeping threads to wake up (if any). - pub(super) fn sub_idle_thread(&self) -> usize { - let old_value = Counters::new(self.value.fetch_sub(ONE_IDLE, Ordering::SeqCst)); + /// + /// See `add_inactive_thread`. + pub(super) fn sub_inactive_thread(&self) -> usize { + let old_value = Counters::new(self.value.fetch_sub(ONE_INACTIVE, Ordering::SeqCst)); debug_assert!( - old_value.raw_idle_threads() > 0, - "sub_idle_thread: old_value {:?} has no idle threads", + old_value.inactive_threads() > 0, + "sub_inactive_thread: old_value {:?} has no inactive threads", old_value, ); debug_assert!( - old_value.sleeping_threads() <= old_value.raw_idle_threads(), - "sub_sleeping_thread: old_value {:?} had {} sleeping threads and {} raw idle threads", + old_value.sleeping_threads() <= old_value.inactive_threads(), + "sub_inactive_thread: old_value {:?} had {} sleeping threads and {} inactive threads", old_value, old_value.sleeping_threads(), - old_value.raw_idle_threads(), + old_value.inactive_threads(), ); - // Current heuristic: whenever an idle thread goes away, if + // Current heuristic: whenever an inactive thread goes away, if // there are any sleeping threads, wake 'em up. let sleeping_threads = old_value.sleeping_threads(); std::cmp::min(sleeping_threads, 2) @@ -157,19 +168,19 @@ impl AtomicCounters { old_value, ); debug_assert!( - old_value.sleeping_threads() <= old_value.raw_idle_threads(), - "sub_sleeping_thread: old_value {:?} had {} sleeping threads and {} raw idle threads", + old_value.sleeping_threads() <= old_value.inactive_threads(), + "sub_sleeping_thread: old_value {:?} had {} sleeping threads and {} inactive threads", old_value, old_value.sleeping_threads(), - old_value.raw_idle_threads(), + old_value.inactive_threads(), ); } #[inline] pub(super) fn try_add_sleeping_thread(&self, old_value: Counters) -> bool { debug_assert!( - old_value.raw_idle_threads() > 0, - "try_add_sleeping_thread: old_value {:?} has no idle threads", + old_value.inactive_threads() > 0, + "try_add_sleeping_thread: old_value {:?} has no inactive threads", old_value, ); debug_assert!( @@ -198,18 +209,20 @@ impl Counters { JobsEventCounter(select_bits(self.word, JEC_SHIFT, JEC_BITS)) } - pub(super) fn raw_idle_threads(self) -> usize { - select_bits(self.word, IDLE_SHIFT, THREADS_BITS) + /// The number of threads that are not actively + /// executing work. They may be idle, sleepy, or asleep. + pub(super) fn inactive_threads(self) -> usize { + select_bits(self.word, INACTIVE_SHIFT, THREADS_BITS) } pub(super) fn awake_but_idle_threads(self) -> usize { debug_assert!( - self.sleeping_threads() <= self.raw_idle_threads(), + self.sleeping_threads() <= self.inactive_threads(), "sleeping threads: {} > raw idle threads {}", self.sleeping_threads(), - self.raw_idle_threads() + self.inactive_threads() ); - self.raw_idle_threads() - self.sleeping_threads() + self.inactive_threads() - self.sleeping_threads() } pub(super) fn sleeping_threads(self) -> usize { @@ -223,7 +236,7 @@ impl std::fmt::Debug for Counters { fmt.debug_struct("Counters") .field("word", &word) .field("jobs", &self.jobs_counter().0) - .field("idle", &self.raw_idle_threads()) + .field("inactive", &self.inactive_threads()) .field("sleeping", &self.sleeping_threads()) .finish() } diff --git a/rayon-core/src/sleep/mod.rs b/rayon-core/src/sleep/mod.rs index 2a40befd5..aac5257f2 100644 --- a/rayon-core/src/sleep/mod.rs +++ b/rayon-core/src/sleep/mod.rs @@ -76,7 +76,7 @@ impl Sleep { latch_addr: latch.addr(), }); - self.counters.add_idle_thread(); + self.counters.add_inactive_thread(); IdleState { worker_index, @@ -94,7 +94,7 @@ impl Sleep { // If we were the last idle thread and other threads are still sleeping, // then we should wake up another thread. - let threads_to_wake = self.counters.sub_idle_thread(); + let threads_to_wake = self.counters.sub_inactive_thread(); self.wake_any_threads(threads_to_wake as u32); } From 5d8996031910bef39b0760852b2391013cb23d0e Mon Sep 17 00:00:00 2001 From: Niko Matsakis Date: Tue, 14 Apr 2020 09:28:59 +0000 Subject: [PATCH 24/33] begin updating README --- rayon-core/src/sleep/README.md | 251 ++++++++++++++++++++------------- 1 file changed, 150 insertions(+), 101 deletions(-) diff --git a/rayon-core/src/sleep/README.md b/rayon-core/src/sleep/README.md index 867496fa8..45ccc2f43 100644 --- a/rayon-core/src/sleep/README.md +++ b/rayon-core/src/sleep/README.md @@ -23,112 +23,161 @@ The `Sleep` struct is embedded into each registry. It performs several functions ## Thread states -An **active** thread is one that is running tasks and doing work. - -An **idle** thread is one that is in a busy loop looking for work. It will be -actively trying to steal from other threads and searching the global injection -queue for jobs. While it does this, it will periodically call back into the -sleep module with information about its progress. - -Towards the end of the idle period, idle threads also become **sleepy**. A -**sleepy** thread is an idle thread that is *about* to go to sleep after it does -one more search (or some other number, potentially). The role of going sleepy is -to prevent a possible race condition, wherein: - -* some thread A attempts to steal work from thread B, but finds that B's queue - is empty; -* thread B posts work, but finds that no workers are sleeping and hence there is - no one to wake; -* thread A decides to go to sleep, and thus never sees the work that B posted. - -This race condition can lead to suboptimal performance because we have no -workers available. - -A **sleeping** thread is one that is blocked and which must be actively awoken. -Sleeping threads will always be blocked on the mutex/condition-variable assigned -to them in the `Sleep` struct. +There are three main thread states: + +* An **active** thread is one that is actively executing a job. +* An **idle** thread is one that is searching for work to do. It will be + trying to steal work or pop work from the global injector queue. +* A **sleeping** thread is one that is blocked on a condition variable, + waiting to be awoken. + +We sometimes refer to the final two states collectively as **inactive**. +Threads begin as idle but transition to idle and finally sleeping when +they're unable to find work to do. + +### Sleepy threads + +There is one other special state worth mentioning. During the idle state, +threads can get **sleepy**. A sleepy thread is still idle, in that it is still +searching for work, but it is *about* to go to sleep after it does one more +search (or some other number, potentially). When a thread enters the sleepy +state, it signals (via the **jobs event counter**, described below) that it is +about to go to sleep. If new work is published, this will lead to the counter +being adjusted. When the thread actually goes to sleep, it will (hopefully, but +not guaranteed) see that the counter has changed and elect not to sleep, but +instead to search again. See the section on the **jobs event counter** for more +details. ## The counters One of the key structs in the sleep module is `AtomicCounters`, found in -`counters.rs`. It packs four counters into one atomically managed value. These -counters fall into two categories: +`counters.rs`. It packs three counters into one atomically managed value: -* **Thread counters**, which track the number of threads in a particular state. -* **Event counters**, which track the number of events that occurred. +* Two **thread counters**, which track the number of threads in a particular state. +* The **jobs event counter**, which is used to signal when new work is available. + It (sort of) tracks the number of jobs posted, but not quite, and it can rollover. ### Thread counters -There are two thread counters, one that tracks **idle** threads and one that -tracks **sleeping** threads. These are incremented and decremented as threads -enter and leave the idle/sleeping state in a fairly straightforward fashion. It -is important, however, that the thread counters are maintained atomically with -the event counters to prevent race conditions between new work being posted and -threads falling asleep. In other words, when new work is posted, it may need to -wake sleeping threads, and so we need a clear ordering whether the *work was -posted first* or the *thread fell asleep first*. - -### Event counters - -There are two event counters, the **jobs event counter** and the **sleepy -event counter**. Event counters, unlike thread counters, are never -decremented (modulo rollover, more on that later). They are simply advanced forward each time some event occurs. - -The jobs event counter is, **conceptually**, incremented each time a new job is posted. The role of this is that, when a thread becomes sleepy, it can read the jobs event counter to see how many jobs were posted up till that point. Then, when it goes to sleep, it can atomically do two things: - -* Verify that the event counter has not changed, and hence no new work was posted; -* Increment the sleepy thread counter, so that if any new work comes later, it - will know that there are sleeping threads that may have to be awoken. - -This simple picture, however, isn't quite what we do -- the problem is two-fold. -First, it would require that each time a new job is posted, we do a write -operation to a single global counter, which would be a performance bottleneck -(how much of one? I'm not sure, measurements would be good!). - -Second, it would not account for rollover -- it is possible, after all, that -enough events have been posted that the counter **has** rolled all the way back -to its original value! - -To address the first point, we keep a separate **sleepy event counter** that -(sort of) tracks how many times threads have become sleepy. Both event counters -begin at zero. Thus, after a new job is posted, we can check if they are still -equal to one another. If so, we can simply post our job, and we can now say that -the job was posted *before* any subsequent thread got sleepy -- therefore, if a -thread becomes sleepy after that, that thread will find the job when it does its -final search for work. - -If however we see that the sleepy event counter (SEC) is **not** equal to the jobs event counter (JEC), that indicates that some thread has become sleepy since work was last posted. This means we have to do more work. What we do, specifically, is to set the JEC to be equal to the SEC, and we do this atomically. - -Meanwhile, the sleepy thread, before it goes to sleep, will read the counters again and check that the JEC has not changed since it got sleepy. So, if new work *was* posted, then the JEC will have changed (to be equal to the SEC), and the sleepy thread will go back to the idle state and search again (and then possibly become sleepy again). - -This is the high-level idea, anyway. Things get subtle when you consider that there are many threads, and also that we must account for rollover. - -## Actual procedure - -Let's document the precise procedure we follow, first. In each case, all the operations happen atomically, which is easy to implement through a compare and swap. - -* To announce a thread is idle: - * IdleThreads += 1 -* To announce a thread found work: - * IdleThreads -= 1 - * If IdleThreads == SleepingThreads, return NeedToWake -* To announce a thread is sleepy: - * If SleepyEventCounter == MAX, set JobsEventCounter = SleepyEventCounter = 0 - * Else, set SleepyEventCounter = SleepyEventCounter + 1 -* To post work: - * If JobsEventCounter != SleepyEventCounter: - * JobsEventCounter = SleepyEventCounter - * NeedToWake = decide(IdleThreads, SleepingThreads) -* To fall asleep: - * If JobsEventCounter > SleepyEventCounter: - * become idle - * SleepingThreads += 1 - -### Accounting for rollover - -Let's consider three threads, A, B, and C: - -* A becomes sleepy, incrementing SleepyEventCounter to get SEC_A (which is MAX) -* In a loop: - * B gets sleepy, reseting SleepyEventCounter to 0 - * B posts work, +There are two thread counters, one that tracks **inactive** threads and one that +tracks **sleeping** threads. From this, one can deduce the number of threads +that are idle by subtracting sleeping threads from inactive threads. We track +the counters in this way because it permits simpler atomic operations. One can +increment the number of sleeping threads (and thus decrease the number of idle +threads) simply by doing one atomic increment, for example. Similarly, one can +decrease the number of sleeping threads (and increase the number of idle +threads) through one atomic decrement. + +These counters are adjusted as follows: + +* When a thread enters the idle state: increment the inactive thread counter. +* When a thread enters the sleeping state: increment the sleeping thread counter. +* When a thread awakens a sleeping thread: decrement the sleeping thread counter. + * Subtle point: the thread that *awakens* the sleeping thread decrements the + counter, not the thread that is *sleeping*. This is because there is a delay + between siganling a thread to wake and the thread actually waking: + decrementing the counter when awakening the thread means that other threads + that may be posting work will see the up-to-date value that much faster. +* When a thread finds work, exiting the idle state: decrement the inactive + thread counter. + +### Jobs event counter + +The final counter is the **jobs event counter**. The role of this counter +is to help sleepy threads detect when new work is posted in a lightweight +fashion. It is important if the counter is even or odd: + +* When the counter is **even**, it means that the last thing to happen was + that new work was published. +* When the counter is **odd**, it means that the last thing to happen was + that some thread got sleepy. + +It works like so: + +* When a thread becomes sleepy, it checks if the counter is even. If so, + it remembers the value (`JEC_OLD`) and increments so that the counter + is odd. +* When a thread publishes work, it checks if the counter is odd. If so, + it increments the counter (potentially rolling it over). +* When a sleepy thread goes to sleep, it reads the counter again. If + the counter value has changed from `JEC_OLD`, then it knows that + work was published during its sleepy time, so it goes back to the idle + state to search again. + +Assuming no rollover, this protocol serves to prevent a race condition +like so: + +* Thread A gets sleepy. +* Thread A searches for work, finds nothing, and decides to go to sleep. +* Thread B posts work, but sees no sleeping threads, and hence no one to wake up. +* Thread A goes to sleep, incrementing the sleeping thread counter. + +The race condition would be prevented because Thread B would have incremented +the JEC, and hence Thread A would not actually go to sleep, but rather return to +search again. + +However, because of rollover, the race condition cannot be completely thwarted. +It is possible, if exceedingly unlikely, that Thread A will get sleepy and read +a value of the JEC. And then, in between, there will be *just enough* activity +from other threads to roll the JEC back over to precisely that old value. + +## Protocol to fall asleep + +The full protocol for a thread to fall asleep is as follows: + +* The thread "announces it is sleepy" by incrementing the JEC if it is even. +* The thread searches for work. If work is found, it becomes active. +* If no work is found, the thread atomically: + * Checks the JEC to see that it hasn't changed. If it has, then the thread + returns to *just before* the "sleepy state" to search again (i.e., it won't + search for a full set of rounds, just a few more times). + * Increments the number of sleeping threads by 1. +* The thread then does one final check for injected jobs (see below). If any + are available, it returns to the 'pre-sleepy' state as if the JEC had changed. +* The thread waits to be signaled. Once signaled, it returns to the idle state. + +### The jobs event counter and deadlock + +As described in the section on the JEC, the main concern around going to sleep +is avoiding a race condition wherein: + +* Thread A looks for work, finds none. +* Thread B posts work but sees no sleeping threads. +* Thread A goes to sleep. + +The JEC protocol largely prevents this, but due to rollover, this prevention is +not complete. It is possible -- if unlikely -- that enough activity occurs for +Thread A to observe the same JEC value that it saw when getting sleepy. If the +new work being published came from *inside* the thread-pool, then this race +condition isn't too harmful. It means that we have fewer workers processing the +work then we should, but we won't deadlock. This seems like an acceptable risk +given that this is unlikely in practice. + +However, if the work was posted as an *external* job, that is a problem. In that +case, it's possible that all of our workers could go to sleep, and the external +job would never get processed. + +To prevent that, the sleeping protocol includes one final check for external +jobs. Note that we are guaranteed to see any such jobs, but the reasoning is +actually pretty subtle. + +The key point is that, if the JEC == JEC_OLD, then there are three possibilities: + +* No intervening operation occurred (i.e., no job was posted). In that case, + going to sleep means that any subsequent job will see (and wake) a sleeping + thread, so no deadlock. +* Rollover occurred and JEC is odd: In this case, the last thing to happen was + that some other thread became sleepy. That thread will see any jobs that were + posted. +* Rollover occurred and JEC is even: In this case, the last thing to happen was + that a job was posted. + +* Thread B will first post the external job into the queue. +* Thread B will then check for sleeping threads, which is a sequentially consistent + read. +* Thread A will increment the number of sleeping threads, which is also SeqCst. +* Thread A then reads the external job. + +XXX this is false -- there is no synchronizes-with relation between a seq-cst +read and the increment on thread A. We could add fences to achieve the desired +effect. \ No newline at end of file From d59dcd6dd7045faefaea492e48aa361845155cf6 Mon Sep 17 00:00:00 2001 From: Niko Matsakis Date: Wed, 15 Apr 2020 10:45:20 +0000 Subject: [PATCH 25/33] simplified JEC scheme --- rayon-core/src/log.rs | 2 +- rayon-core/src/sleep/README.md | 109 +++++++++++++++---------------- rayon-core/src/sleep/counters.rs | 11 +--- rayon-core/src/sleep/mod.rs | 32 ++++----- 4 files changed, 69 insertions(+), 85 deletions(-) diff --git a/rayon-core/src/log.rs b/rayon-core/src/log.rs index 99695d1a0..e1ff827df 100644 --- a/rayon-core/src/log.rs +++ b/rayon-core/src/log.rs @@ -56,7 +56,7 @@ pub(super) enum Event { /// Indicates that an idle worker is getting sleepy. `sleepy_counter` is the internal /// sleep state that we saw at the time. - ThreadSleepy { worker: usize, jobs_counter: u16 }, + ThreadSleepy { worker: usize, jobs_counter: usize }, /// Indicates that the thread's attempt to fall asleep was /// interrupted because the latch was set. (This is not, in and of diff --git a/rayon-core/src/sleep/README.md b/rayon-core/src/sleep/README.md index 45ccc2f43..d2aebe06b 100644 --- a/rayon-core/src/sleep/README.md +++ b/rayon-core/src/sleep/README.md @@ -83,26 +83,17 @@ These counters are adjusted as follows: ### Jobs event counter -The final counter is the **jobs event counter**. The role of this counter -is to help sleepy threads detect when new work is posted in a lightweight -fashion. It is important if the counter is even or odd: - -* When the counter is **even**, it means that the last thing to happen was - that new work was published. -* When the counter is **odd**, it means that the last thing to happen was - that some thread got sleepy. - -It works like so: - -* When a thread becomes sleepy, it checks if the counter is even. If so, - it remembers the value (`JEC_OLD`) and increments so that the counter - is odd. -* When a thread publishes work, it checks if the counter is odd. If so, - it increments the counter (potentially rolling it over). -* When a sleepy thread goes to sleep, it reads the counter again. If - the counter value has changed from `JEC_OLD`, then it knows that - work was published during its sleepy time, so it goes back to the idle - state to search again. +The final counter is the **jobs event counter**. The role of this counter is to +help sleepy threads detect when new work is posted in a lightweight fashion. The +counter is incremented every time a new job is posted -- naturally, it can also +rollover if there have been enough jobs posted. + +The counter is used as follows: + +* When a thread gets **sleepy**, it reads the current value of the counter. +* Later, before it goes to sleep, it checks if the counter has changed. + If it has, that indicates that work was posted but we somehow missed it + while searching. We'll go back and search again. Assuming no rollover, this protocol serves to prevent a race condition like so: @@ -112,25 +103,29 @@ like so: * Thread B posts work, but sees no sleeping threads, and hence no one to wake up. * Thread A goes to sleep, incrementing the sleeping thread counter. -The race condition would be prevented because Thread B would have incremented -the JEC, and hence Thread A would not actually go to sleep, but rather return to -search again. - However, because of rollover, the race condition cannot be completely thwarted. It is possible, if exceedingly unlikely, that Thread A will get sleepy and read a value of the JEC. And then, in between, there will be *just enough* activity -from other threads to roll the JEC back over to precisely that old value. +from other threads to roll the JEC back over to precisely that old value. We +have an extra check in the protocol to prevent deadlock in that (rather +unlikely) case. -## Protocol to fall asleep +## Protocol for a worker thread to fall asleep The full protocol for a thread to fall asleep is as follows: -* The thread "announces it is sleepy" by incrementing the JEC if it is even. -* The thread searches for work. If work is found, it becomes active. +* After completing all its jobs, the worker goes idle and begins to + search for work. As it searches, it counts "rounds". In each round, + it searches all other work threads' queues, plus the 'injector queue' for + work injected from the outside. If work is found in this search, the thread + becomes active again and hence restarts this protocol from the top. +* After a certain number of rounds, the thread "gets sleepy" and reads the JEC. + It does one more search for work. * If no work is found, the thread atomically: - * Checks the JEC to see that it hasn't changed. If it has, then the thread - returns to *just before* the "sleepy state" to search again (i.e., it won't - search for a full set of rounds, just a few more times). + * Checks the JEC to see that it hasn't changed. + * If it has, then the thread returns to *just before* the "sleepy state" to + search again (i.e., it won't search for a full set of rounds, just a few + more times). * Increments the number of sleeping threads by 1. * The thread then does one final check for injected jobs (see below). If any are available, it returns to the 'pre-sleepy' state as if the JEC had changed. @@ -155,29 +150,29 @@ given that this is unlikely in practice. However, if the work was posted as an *external* job, that is a problem. In that case, it's possible that all of our workers could go to sleep, and the external -job would never get processed. - -To prevent that, the sleeping protocol includes one final check for external -jobs. Note that we are guaranteed to see any such jobs, but the reasoning is -actually pretty subtle. - -The key point is that, if the JEC == JEC_OLD, then there are three possibilities: - -* No intervening operation occurred (i.e., no job was posted). In that case, - going to sleep means that any subsequent job will see (and wake) a sleeping - thread, so no deadlock. -* Rollover occurred and JEC is odd: In this case, the last thing to happen was - that some other thread became sleepy. That thread will see any jobs that were - posted. -* Rollover occurred and JEC is even: In this case, the last thing to happen was - that a job was posted. - -* Thread B will first post the external job into the queue. -* Thread B will then check for sleeping threads, which is a sequentially consistent - read. -* Thread A will increment the number of sleeping threads, which is also SeqCst. -* Thread A then reads the external job. - -XXX this is false -- there is no synchronizes-with relation between a seq-cst -read and the increment on thread A. We could add fences to achieve the desired -effect. \ No newline at end of file +job would never get processed. To prevent that, the sleeping protocol includes +one final check to see if the injector queue is empty before fully falling +asleep. Note that this final check occurs **after** the number of sleeping +threads has been incremented. We are not concerned therefore with races against +injections that occur after that increment, only before. + +What follows is a "proof sketch" that the protocol is deadlock free. We model +two relevant bits of memory, the job injector queue J and the atomic counters C. + +Consider the actions of the injecting thread: + +* PushJob: Job is injected, which can be modeled as an atomic write to J with release semantics. +* IncJec: The JEC is incremented, which can be modeled as an atomic exchange to C with acquire-release semantics. + +Meanwhile, the sleepy thread does the following: + +* IncSleepers: The number of sleeping threads is incremented, which is atomic exchange to C with acquire-release semantics. +* ReadJob: We look to see if the queue is empty, which is a read of J with acquire semantics. + +Both IncJec and IncSleepers modify the same memory location, and hence they must be fully ordered. + +* If IncSleepers came first, there is no problem, because the injecting thread + knows that everyone is asleep and it will wake up a thread. +* If IncJec came first, then it "synchronizes with" IncSleepers. + * Therefore, PushJob "happens before" ReadJob, and so the write will be visible during + this final check, and the thread will not go to sleep. diff --git a/rayon-core/src/sleep/counters.rs b/rayon-core/src/sleep/counters.rs index 031b33248..3f8c4a454 100644 --- a/rayon-core/src/sleep/counters.rs +++ b/rayon-core/src/sleep/counters.rs @@ -27,17 +27,9 @@ pub(super) struct JobsEventCounter(usize); impl JobsEventCounter { pub(super) const MAX: JobsEventCounter = JobsEventCounter(JEC_MAX); - fn as_usize(self) -> usize { + pub(super) fn as_usize(self) -> usize { self.0 } - - /// Returns true if there were sleepy workers pending when this reading was - /// taken. This is determined by checking whether the value is odd (sleepy - /// workers) or even (work published since last sleepy worker got sleepy, - /// though they may not have seen it yet). - pub(super) fn is_sleepy(self) -> bool { - (self.as_usize() & 1) != 0 - } } /// Constant that can be added to add one sleeping thread. @@ -121,6 +113,7 @@ impl AtomicCounters { /// * If a thread is publishing work, and the JEC is odd, then it will attempt /// to increment to an event value. pub(super) fn try_increment_jobs_event_counter(&self, old_value: Counters) -> bool { + // FIXME -- we should remove the `MAX` constant and just let rollover happen naturally let new_value = if old_value.jobs_counter() == JobsEventCounter::MAX { Counters::new(old_value.word & ZERO_JEC_MASK) } else { diff --git a/rayon-core/src/sleep/mod.rs b/rayon-core/src/sleep/mod.rs index aac5257f2..294769f6c 100644 --- a/rayon-core/src/sleep/mod.rs +++ b/rayon-core/src/sleep/mod.rs @@ -123,18 +123,13 @@ impl Sleep { #[cold] fn announce_sleepy(&self, worker_index: usize) -> JobsEventCounter { - loop { - let counters = self.counters.load(Ordering::Relaxed); - let jobs_counter = counters.jobs_counter(); - if jobs_counter.is_sleepy() || self.counters.try_increment_jobs_event_counter(counters) - { - self.logger.log(|| ThreadSleepy { - worker: worker_index, - jobs_counter: 0, - }); - return jobs_counter; - } - } + let counters = self.counters.load(Ordering::SeqCst); + let jobs_counter = counters.jobs_counter(); + self.logger.log(|| ThreadSleepy { + worker: worker_index, + jobs_counter: jobs_counter.as_usize(), + }); + jobs_counter } #[cold] @@ -173,14 +168,17 @@ impl Sleep { loop { let counters = self.counters.load(Ordering::SeqCst); + + // Check if the JEC has changed since we got sleepy. if counters.jobs_counter() != idle_state.jobs_counter { + // JEC has changed, so a new job was posted, but for some reason + // we didn't see it. We should return to just before the SLEEPY + // state so we can do another search and (if we fail to find + // work) go back to sleep. self.logger.log(|| ThreadSleepInterruptedByJob { worker: worker_index, }); - // A new job was posted. We should return to just - // before the SLEEPY state so we can do another search - // and (if we fail to find work) go back to sleep. idle_state.wake_partly(); latch.wake_up(); return; @@ -300,9 +298,7 @@ impl Sleep { // with which we exit the loop thus corresponds to a state when loop { counters = self.counters.load(Ordering::SeqCst); - if !counters.jobs_counter().is_sleepy() { - break; - } else if self.counters.try_increment_jobs_event_counter(counters) { + if self.counters.try_increment_jobs_event_counter(counters) { break; } } From 490a39f512f39c0c5896adaa9dab6d57c2cf83d7 Mon Sep 17 00:00:00 2001 From: Niko Matsakis Date: Wed, 15 Apr 2020 10:57:25 +0000 Subject: [PATCH 26/33] prep work for usize JEC --- rayon-core/src/sleep/counters.rs | 66 +++++++++++++------------------- rayon-core/src/sleep/mod.rs | 15 ++------ 2 files changed, 30 insertions(+), 51 deletions(-) diff --git a/rayon-core/src/sleep/counters.rs b/rayon-core/src/sleep/counters.rs index 3f8c4a454..e9120d3ac 100644 --- a/rayon-core/src/sleep/counters.rs +++ b/rayon-core/src/sleep/counters.rs @@ -25,52 +25,40 @@ pub(super) struct Counters { pub(super) struct JobsEventCounter(usize); impl JobsEventCounter { - pub(super) const MAX: JobsEventCounter = JobsEventCounter(JEC_MAX); + pub(super) const DUMMY: JobsEventCounter = JobsEventCounter(std::usize::MAX); pub(super) fn as_usize(self) -> usize { self.0 } } -/// Constant that can be added to add one sleeping thread. -const ONE_SLEEPING: u64 = 0x0000_0000_0000_0001; - -/// Constant that can be added to add one inactive thread. -/// An inactive thread is either idle, sleepy, or sleeping. -const ONE_INACTIVE: u64 = 0x0000_0000_0001_0000; - -/// Constant that can be added to add one to the JEC. -const ONE_JEC: u64 = 0x0000_0001_0000_0000; - -/// Mask to zero out the JEC (but retain everything else). -const ZERO_JEC_MASK: u64 = 0x0000_0000_FFFF_FFFF; +/// Number of bits used for the thread counters. +const THREADS_BITS: u64 = 10; /// Bits to shift to select the sleeping threads /// (used with `select_bits`). -const SLEEPING_SHIFT: u64 = 0; +const SLEEPING_SHIFT: u64 = 0 * THREADS_BITS; /// Bits to shift to select the inactive threads /// (used with `select_bits`). -const INACTIVE_SHIFT: u64 = 1 * 16; +const INACTIVE_SHIFT: u64 = 1 * THREADS_BITS; /// Bits to shift to select the JEC /// (use JOBS_BITS). -const JEC_SHIFT: u64 = 2 * 16; - -/// Max value for the jobs event counter. -const JEC_MAX: usize = 0xFFFF; +const JEC_SHIFT: u64 = 2 * THREADS_BITS; /// Max value for the thread counters. -const THREADS_MAX: usize = 0xFFFF; +const THREADS_MAX: usize = (1 << THREADS_BITS) - 1; -/// Mask to select the value of a thread counter -/// (sleepy, inactive), after shifting (used with -/// `select_bits`) -const THREADS_BITS: usize = 0xFFFF; +/// Constant that can be added to add one sleeping thread. +const ONE_SLEEPING: u64 = 1; + +/// Constant that can be added to add one inactive thread. +/// An inactive thread is either idle, sleepy, or sleeping. +const ONE_INACTIVE: u64 = 1 << INACTIVE_SHIFT; -/// Mask to select the JEC, after -/// shifting (used with `select_bits`) -const JEC_BITS: usize = std::usize::MAX; +/// Constant that can be added to add one to the JEC. +const ONE_JEC: u64 = 1 << JEC_SHIFT; impl AtomicCounters { pub(super) fn new() -> AtomicCounters { @@ -112,14 +100,8 @@ impl AtomicCounters { /// to increment to an odd value. /// * If a thread is publishing work, and the JEC is odd, then it will attempt /// to increment to an event value. - pub(super) fn try_increment_jobs_event_counter(&self, old_value: Counters) -> bool { - // FIXME -- we should remove the `MAX` constant and just let rollover happen naturally - let new_value = if old_value.jobs_counter() == JobsEventCounter::MAX { - Counters::new(old_value.word & ZERO_JEC_MASK) - } else { - Counters::new(old_value.word + ONE_JEC) - }; - self.try_exchange(old_value, new_value, Ordering::SeqCst) + pub(super) fn increment_and_read_jobs_event_counter(&self) -> Counters { + Counters::new(self.value.fetch_add(ONE_JEC, Ordering::SeqCst)) } /// Subtracts an inactive thread. This cannot fail. It is invoked @@ -189,8 +171,12 @@ impl AtomicCounters { } } -fn select_bits(word: u64, shift: u64, bits: usize) -> usize { - ((word >> shift) as usize) & bits +fn select_thread(word: u64, shift: u64) -> usize { + ((word >> shift) as usize) & THREADS_MAX +} + +fn select_jec(word: u64) -> usize { + (word >> JEC_SHIFT) as usize } impl Counters { @@ -199,13 +185,13 @@ impl Counters { } pub(super) fn jobs_counter(self) -> JobsEventCounter { - JobsEventCounter(select_bits(self.word, JEC_SHIFT, JEC_BITS)) + JobsEventCounter(select_jec(self.word)) } /// The number of threads that are not actively /// executing work. They may be idle, sleepy, or asleep. pub(super) fn inactive_threads(self) -> usize { - select_bits(self.word, INACTIVE_SHIFT, THREADS_BITS) + select_thread(self.word, INACTIVE_SHIFT) } pub(super) fn awake_but_idle_threads(self) -> usize { @@ -219,7 +205,7 @@ impl Counters { } pub(super) fn sleeping_threads(self) -> usize { - select_bits(self.word, SLEEPING_SHIFT, THREADS_BITS) + select_thread(self.word, SLEEPING_SHIFT) } } diff --git a/rayon-core/src/sleep/mod.rs b/rayon-core/src/sleep/mod.rs index 294769f6c..4db30c58a 100644 --- a/rayon-core/src/sleep/mod.rs +++ b/rayon-core/src/sleep/mod.rs @@ -81,7 +81,7 @@ impl Sleep { IdleState { worker_index, rounds: 0, - jobs_counter: JobsEventCounter::MAX, + jobs_counter: JobsEventCounter::DUMMY, } } @@ -291,17 +291,10 @@ impl Sleep { /// Common helper for `new_injected_jobs` and `new_internal_jobs`. #[inline] fn new_jobs(&self, source_worker_index: usize, num_jobs: u32, queue_was_empty: bool) { - let mut counters; - // Read the counters and -- if sleepy workers have announced themselves // -- announce that there is now work available. The final value of `counters` // with which we exit the loop thus corresponds to a state when - loop { - counters = self.counters.load(Ordering::SeqCst); - if self.counters.try_increment_jobs_event_counter(counters) { - break; - } - } + let counters = self.counters.increment_and_read_jobs_event_counter(); let num_awake_but_idle = counters.awake_but_idle_threads(); let num_sleepers = counters.sleeping_threads(); @@ -379,11 +372,11 @@ impl Sleep { impl IdleState { fn wake_fully(&mut self) { self.rounds = 0; - self.jobs_counter = JobsEventCounter::MAX; + self.jobs_counter = JobsEventCounter::DUMMY; } fn wake_partly(&mut self) { self.rounds = ROUNDS_UNTIL_SLEEPY; - self.jobs_counter = JobsEventCounter::MAX; + self.jobs_counter = JobsEventCounter::DUMMY; } } From 71d214efa48941f820a509519c90f0237788857c Mon Sep 17 00:00:00 2001 From: Niko Matsakis Date: Wed, 15 Apr 2020 11:01:41 +0000 Subject: [PATCH 27/33] change to AtomicUsize --- rayon-core/src/sleep/counters.rs | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/rayon-core/src/sleep/counters.rs b/rayon-core/src/sleep/counters.rs index e9120d3ac..1a6d5a6b0 100644 --- a/rayon-core/src/sleep/counters.rs +++ b/rayon-core/src/sleep/counters.rs @@ -1,4 +1,4 @@ -use std::sync::atomic::{AtomicU64, Ordering}; +use std::sync::atomic::{AtomicUsize, Ordering}; pub(super) struct AtomicCounters { /// Packs together a number of counters. @@ -10,12 +10,12 @@ pub(super) struct AtomicCounters { /// - The bits 0x4444 are the number of **sleeping threads**. /// /// See the struct `Counters` below. - value: AtomicU64, + value: AtomicUsize, } #[derive(Copy, Clone)] pub(super) struct Counters { - word: u64, + word: usize, } /// A value read from the **Jobs Event Counter**. @@ -33,37 +33,37 @@ impl JobsEventCounter { } /// Number of bits used for the thread counters. -const THREADS_BITS: u64 = 10; +const THREADS_BITS: usize = 10; /// Bits to shift to select the sleeping threads /// (used with `select_bits`). -const SLEEPING_SHIFT: u64 = 0 * THREADS_BITS; +const SLEEPING_SHIFT: usize = 0 * THREADS_BITS; /// Bits to shift to select the inactive threads /// (used with `select_bits`). -const INACTIVE_SHIFT: u64 = 1 * THREADS_BITS; +const INACTIVE_SHIFT: usize = 1 * THREADS_BITS; /// Bits to shift to select the JEC /// (use JOBS_BITS). -const JEC_SHIFT: u64 = 2 * THREADS_BITS; +const JEC_SHIFT: usize = 2 * THREADS_BITS; /// Max value for the thread counters. const THREADS_MAX: usize = (1 << THREADS_BITS) - 1; /// Constant that can be added to add one sleeping thread. -const ONE_SLEEPING: u64 = 1; +const ONE_SLEEPING: usize = 1; /// Constant that can be added to add one inactive thread. /// An inactive thread is either idle, sleepy, or sleeping. -const ONE_INACTIVE: u64 = 1 << INACTIVE_SHIFT; +const ONE_INACTIVE: usize = 1 << INACTIVE_SHIFT; /// Constant that can be added to add one to the JEC. -const ONE_JEC: u64 = 1 << JEC_SHIFT; +const ONE_JEC: usize = 1 << JEC_SHIFT; impl AtomicCounters { pub(super) fn new() -> AtomicCounters { AtomicCounters { - value: AtomicU64::new(0), + value: AtomicUsize::new(0), } } @@ -171,16 +171,16 @@ impl AtomicCounters { } } -fn select_thread(word: u64, shift: u64) -> usize { +fn select_thread(word: usize, shift: usize) -> usize { ((word >> shift) as usize) & THREADS_MAX } -fn select_jec(word: u64) -> usize { +fn select_jec(word: usize) -> usize { (word >> JEC_SHIFT) as usize } impl Counters { - fn new(word: u64) -> Counters { + fn new(word: usize) -> Counters { Counters { word } } From 700061b16492128900aac452a9357410f5713f0f Mon Sep 17 00:00:00 2001 From: Niko Matsakis Date: Wed, 15 Apr 2020 13:29:11 +0000 Subject: [PATCH 28/33] document bit layout correctly --- rayon-core/src/sleep/counters.rs | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/rayon-core/src/sleep/counters.rs b/rayon-core/src/sleep/counters.rs index 1a6d5a6b0..5111cbd8b 100644 --- a/rayon-core/src/sleep/counters.rs +++ b/rayon-core/src/sleep/counters.rs @@ -1,15 +1,17 @@ use std::sync::atomic::{AtomicUsize, Ordering}; pub(super) struct AtomicCounters { - /// Packs together a number of counters. + /// Packs together a number of counters. The counters are ordered as + /// follows, from least to most significant bits (here, we assuming + /// that [`THREADS_BITS`] is equal to 10): /// - /// Consider the value `0x11112222_3333_4444`: + /// * Bits 0..10: Stores the number of **sleeping threads** + /// * Bits 10..20: Stores the number of **inactive threads** + /// * Bits 20..: Stores the **job event counter** (JEC) /// - /// - The bits 0x11112222 are the **jobs event counter**. - /// - The bits 0x3333 are the number of **inactive threads**. - /// - The bits 0x4444 are the number of **sleeping threads**. - /// - /// See the struct `Counters` below. + /// This uses 10 bits ([`THREADS_BITS`]) to encode the number of threads. Note + /// that the total number of bits (and hence the number of bits used for the + /// JEC) will depend on whether we are using a 32- or 64-bit architecture. value: AtomicUsize, } From 76ce5f5331c6dbf6a7d942cb1f81d01d5ec0e977 Mon Sep 17 00:00:00 2001 From: Josh Stone Date: Wed, 6 May 2020 12:49:55 -0700 Subject: [PATCH 29/33] Add #[inline] to tiny counter/latch methods --- rayon-core/src/latch.rs | 10 ++++++++++ rayon-core/src/registry.rs | 2 ++ rayon-core/src/sleep/counters.rs | 11 +++++++++++ 3 files changed, 23 insertions(+) diff --git a/rayon-core/src/latch.rs b/rayon-core/src/latch.rs index f933a0b98..0965bb95a 100644 --- a/rayon-core/src/latch.rs +++ b/rayon-core/src/latch.rs @@ -70,6 +70,7 @@ pub(super) struct CoreLatch { } impl CoreLatch { + #[inline] fn new() -> Self { Self { state: AtomicUsize::new(0), @@ -78,6 +79,7 @@ impl CoreLatch { /// Returns the address of this core latch as an integer. Used /// for logging. + #[inline] pub(super) fn addr(&self) -> usize { self as *const CoreLatch as usize } @@ -85,6 +87,7 @@ impl CoreLatch { /// Invoked by owning thread as it prepares to sleep. Returns true /// if the owning thread may proceed to fall asleep, false if the /// latch was set in the meantime. + #[inline] pub(super) fn get_sleepy(&self) -> bool { self.state .compare_exchange(UNSET, SLEEPY, Ordering::SeqCst, Ordering::Relaxed) @@ -94,6 +97,7 @@ impl CoreLatch { /// Invoked by owning thread as it falls asleep sleep. Returns /// true if the owning thread should block, or false if the latch /// was set in the meantime. + #[inline] pub(super) fn fall_asleep(&self) -> bool { self.state .compare_exchange(SLEEPY, SLEEPING, Ordering::SeqCst, Ordering::Relaxed) @@ -103,6 +107,7 @@ impl CoreLatch { /// Invoked by owning thread as it falls asleep sleep. Returns /// true if the owning thread should block, or false if the latch /// was set in the meantime. + #[inline] pub(super) fn wake_up(&self) { if !self.probe() { let _ = @@ -117,12 +122,14 @@ impl CoreLatch { /// This is private because, typically, setting a latch involves /// doing some wakeups; those are encapsulated in the surrounding /// latch code. + #[inline] fn set(&self) -> bool { let old_state = self.state.swap(SET, Ordering::AcqRel); old_state == SLEEPING } /// Test if this latch has been set. + #[inline] pub(super) fn probe(&self) -> bool { self.state.load(Ordering::Acquire) == SET } @@ -156,6 +163,7 @@ impl<'r> SpinLatch<'r> { /// Creates a new spin latch for cross-threadpool blocking. Notably, we /// need to make sure the registry is kept alive after setting, so we can /// safely call the notification. + #[inline] pub(super) fn cross(thread: &'r WorkerThread) -> SpinLatch<'r> { SpinLatch { cross: true, @@ -163,6 +171,7 @@ impl<'r> SpinLatch<'r> { } } + #[inline] pub(super) fn probe(&self) -> bool { self.core_latch.probe() } @@ -323,6 +332,7 @@ impl<'a, L> Latch for &'a L where L: Latch, { + #[inline] fn set(&self) { L::set(self); } diff --git a/rayon-core/src/registry.rs b/rayon-core/src/registry.rs index 0a1a8b2d0..53032cca3 100644 --- a/rayon-core/src/registry.rs +++ b/rayon-core/src/registry.rs @@ -640,6 +640,7 @@ impl WorkerThread { } /// Returns the registry that owns this worker thread. + #[inline] pub(super) fn registry(&self) -> &Arc { &self.registry } @@ -743,6 +744,7 @@ impl WorkerThread { mem::forget(abort_guard); // successful execution, do not abort } + #[inline] pub(super) unsafe fn execute(&self, job: JobRef) { job.execute(); } diff --git a/rayon-core/src/sleep/counters.rs b/rayon-core/src/sleep/counters.rs index 5111cbd8b..5c94bc6b1 100644 --- a/rayon-core/src/sleep/counters.rs +++ b/rayon-core/src/sleep/counters.rs @@ -63,6 +63,7 @@ const ONE_INACTIVE: usize = 1 << INACTIVE_SHIFT; const ONE_JEC: usize = 1 << JEC_SHIFT; impl AtomicCounters { + #[inline] pub(super) fn new() -> AtomicCounters { AtomicCounters { value: AtomicUsize::new(0), @@ -72,6 +73,7 @@ impl AtomicCounters { /// Load and return the current value of the various counters. /// This value can then be given to other method which will /// attempt to update the counters via compare-and-swap. + #[inline] pub(super) fn load(&self, ordering: Ordering) -> Counters { Counters::new(self.value.load(ordering)) } @@ -102,6 +104,7 @@ impl AtomicCounters { /// to increment to an odd value. /// * If a thread is publishing work, and the JEC is odd, then it will attempt /// to increment to an event value. + #[inline] pub(super) fn increment_and_read_jobs_event_counter(&self) -> Counters { Counters::new(self.value.fetch_add(ONE_JEC, Ordering::SeqCst)) } @@ -111,6 +114,7 @@ impl AtomicCounters { /// number of sleeping threads to wake up (if any). /// /// See `add_inactive_thread`. + #[inline] pub(super) fn sub_inactive_thread(&self) -> usize { let old_value = Counters::new(self.value.fetch_sub(ONE_INACTIVE, Ordering::SeqCst)); debug_assert!( @@ -173,29 +177,35 @@ impl AtomicCounters { } } +#[inline] fn select_thread(word: usize, shift: usize) -> usize { ((word >> shift) as usize) & THREADS_MAX } +#[inline] fn select_jec(word: usize) -> usize { (word >> JEC_SHIFT) as usize } impl Counters { + #[inline] fn new(word: usize) -> Counters { Counters { word } } + #[inline] pub(super) fn jobs_counter(self) -> JobsEventCounter { JobsEventCounter(select_jec(self.word)) } /// The number of threads that are not actively /// executing work. They may be idle, sleepy, or asleep. + #[inline] pub(super) fn inactive_threads(self) -> usize { select_thread(self.word, INACTIVE_SHIFT) } + #[inline] pub(super) fn awake_but_idle_threads(self) -> usize { debug_assert!( self.sleeping_threads() <= self.inactive_threads(), @@ -206,6 +216,7 @@ impl Counters { self.inactive_threads() - self.sleeping_threads() } + #[inline] pub(super) fn sleeping_threads(self) -> usize { select_thread(self.word, SLEEPING_SHIFT) } From a0efb4abb5bc3a2ee0ed812392a2b7386dc2dd38 Mon Sep 17 00:00:00 2001 From: Niko Matsakis Date: Wed, 13 May 2020 10:37:02 +0000 Subject: [PATCH 30/33] transition to even/odd scheme This avoids the need to increment the JEC on every new job while still, I believe, avoiding deadlock. --- rayon-core/src/sleep/README.md | 125 ++++++++++++++++++++----------- rayon-core/src/sleep/counters.rs | 50 ++++++++++--- rayon-core/src/sleep/mod.rs | 16 +++- 3 files changed, 136 insertions(+), 55 deletions(-) diff --git a/rayon-core/src/sleep/README.md b/rayon-core/src/sleep/README.md index d2aebe06b..c62c3975d 100644 --- a/rayon-core/src/sleep/README.md +++ b/rayon-core/src/sleep/README.md @@ -10,7 +10,7 @@ extracted from the RFC and meant to be kept up to date. [Rayon RFC #5]: https://github.com/rayon-rs/rfcs/pull/5 [video walkthrough]: https://youtu.be/HvmQsE5M4cY -## The `Sleep` struct +# The `Sleep` struct The `Sleep` struct is embedded into each registry. It performs several functions: @@ -21,7 +21,7 @@ The `Sleep` struct is embedded into each registry. It performs several functions events occur, and it will go and wake the appropriate threads if they are sleeping. -## Thread states +# Thread states There are three main thread states: @@ -35,7 +35,7 @@ We sometimes refer to the final two states collectively as **inactive**. Threads begin as idle but transition to idle and finally sleeping when they're unable to find work to do. -### Sleepy threads +## Sleepy threads There is one other special state worth mentioning. During the idle state, threads can get **sleepy**. A sleepy thread is still idle, in that it is still @@ -48,7 +48,7 @@ not guaranteed) see that the counter has changed and elect not to sleep, but instead to search again. See the section on the **jobs event counter** for more details. -## The counters +# The counters One of the key structs in the sleep module is `AtomicCounters`, found in `counters.rs`. It packs three counters into one atomically managed value: @@ -57,7 +57,7 @@ One of the key structs in the sleep module is `AtomicCounters`, found in * The **jobs event counter**, which is used to signal when new work is available. It (sort of) tracks the number of jobs posted, but not quite, and it can rollover. -### Thread counters +## Thread counters There are two thread counters, one that tracks **inactive** threads and one that tracks **sleeping** threads. From this, one can deduce the number of threads @@ -81,36 +81,50 @@ These counters are adjusted as follows: * When a thread finds work, exiting the idle state: decrement the inactive thread counter. -### Jobs event counter +## Jobs event counter The final counter is the **jobs event counter**. The role of this counter is to -help sleepy threads detect when new work is posted in a lightweight fashion. The -counter is incremented every time a new job is posted -- naturally, it can also -rollover if there have been enough jobs posted. +help sleepy threads detect when new work is posted in a lightweight fashion. In +its simplest form, we would simply have a counter that gets incremented each +time a new job is posted. This way, when a thread gets sleepy, it could read the +counter, and then compare to see if the value has changed before it actually +goes to sleep. But this [turns out to be too expensive] in practice, so we use a +somewhat more complex scheme. -The counter is used as follows: +[turns out to be too expensive]: https://github.com/rayon-rs/rayon/pull/746#issuecomment-624802747 -* When a thread gets **sleepy**, it reads the current value of the counter. -* Later, before it goes to sleep, it checks if the counter has changed. - If it has, that indicates that work was posted but we somehow missed it - while searching. We'll go back and search again. +The idea is that the counter toggles between two states, depending on whether +its value is even or odd (or, equivalently, on the value of its low bit): -Assuming no rollover, this protocol serves to prevent a race condition -like so: +* Even -- If the low bit is zero, then it means that there has been no new work + since the last thread got sleepy. +* Odd -- If the low bit is one, then it means that new work was posted since + the last thread got sleepy. -* Thread A gets sleepy. -* Thread A searches for work, finds nothing, and decides to go to sleep. -* Thread B posts work, but sees no sleeping threads, and hence no one to wake up. -* Thread A goes to sleep, incrementing the sleeping thread counter. +### New work is posted -However, because of rollover, the race condition cannot be completely thwarted. -It is possible, if exceedingly unlikely, that Thread A will get sleepy and read -a value of the JEC. And then, in between, there will be *just enough* activity -from other threads to roll the JEC back over to precisely that old value. We -have an extra check in the protocol to prevent deadlock in that (rather -unlikely) case. +When new work is posted, we check the value of the counter: if it is even, +then we increment it by one, so that it becomes odd. -## Protocol for a worker thread to fall asleep +### Worker thread gets sleepy + +When a worker thread gets sleepy, it will read the value of the counter. If the +counter is odd, it will increment the counter so that it is even. Either way, it +remembers the final value of the counter. The final value will be used later, +when the thread is going to sleep. If at that time the counter has not changed, +then we can assume no new jobs have been posted (though note the remote +possibility of rollover, discussed in detail below). + +# Protocol for a worker thread to post work + +The full protocol for a thread to post work is as follows + +* If the work is posted into the injection queue, then execute a seq-cst fence (see below). +* Load the counters, incrementing the JEC if it is even so that it is odd. +* Check if there are idle threads available to handle this new job. If not, + and there are sleeping threads, then wake one or more threads. + +# Protocol for a worker thread to fall asleep The full protocol for a thread to fall asleep is as follows: @@ -119,19 +133,20 @@ The full protocol for a thread to fall asleep is as follows: it searches all other work threads' queues, plus the 'injector queue' for work injected from the outside. If work is found in this search, the thread becomes active again and hence restarts this protocol from the top. -* After a certain number of rounds, the thread "gets sleepy" and reads the JEC. - It does one more search for work. +* After a certain number of rounds, the thread "gets sleepy" and executes `get_sleepy` + above, remembering the `final_value` of the JEC. It does one more search for work. * If no work is found, the thread atomically: - * Checks the JEC to see that it hasn't changed. - * If it has, then the thread returns to *just before* the "sleepy state" to - search again (i.e., it won't search for a full set of rounds, just a few - more times). + * Checks the JEC to see that it has not changed from `final_value`. + * If it has, then the thread goes back to searchin for work. We reset to + just before we got sleepy, so that we will do one more search + before attending to sleep again (rather than searching for many rounds). * Increments the number of sleeping threads by 1. +* The thread then executes a seq-cst fence operation (see below). * The thread then does one final check for injected jobs (see below). If any are available, it returns to the 'pre-sleepy' state as if the JEC had changed. * The thread waits to be signaled. Once signaled, it returns to the idle state. -### The jobs event counter and deadlock +# The jobs event counter and deadlock As described in the section on the JEC, the main concern around going to sleep is avoiding a race condition wherein: @@ -156,23 +171,49 @@ asleep. Note that this final check occurs **after** the number of sleeping threads has been incremented. We are not concerned therefore with races against injections that occur after that increment, only before. +Unfortunately, there is one rather subtle point concerning this final check: +we wish to avoid the possibility that: + +* work is pushed into the injection queue by an outside thread X, +* the sleepy thread S sees the JEC but it has rolled over and is equal +* the sleepy thread S reads the injection queue but does not see the work posted by X. + +This is possible because the C++ memory model typically offers guarantees of the +form "if you see the access A, then you must see those other accesses" -- but it +doesn't guarantee that you will see the access A (i.e., if you think of +processors with independent caches, you may be operating on very out of date +cache state). + +## Using seq-cst fences to prevent deadlock + +To overcome this problem, we have inserted two sequentially consistent fence +operations into the protocols above: + +* One fence occurs after work is posted into the injection queue, but before the + counters are read (including the number of sleeping threads). + * Note that no fence is needed for work posted to internal queues, since it is ok + to overlook work in that case. +* One fence occurs after the number of sleeping threads is incremented, but + before the injection queue is read. + +### Proof sketch + What follows is a "proof sketch" that the protocol is deadlock free. We model two relevant bits of memory, the job injector queue J and the atomic counters C. Consider the actions of the injecting thread: * PushJob: Job is injected, which can be modeled as an atomic write to J with release semantics. -* IncJec: The JEC is incremented, which can be modeled as an atomic exchange to C with acquire-release semantics. +* PushFence: A sequentially consistent fence is executed. +* ReadSleepers: The counters C are read (they may also be incremented, but we just consider the read that comes first). Meanwhile, the sleepy thread does the following: -* IncSleepers: The number of sleeping threads is incremented, which is atomic exchange to C with acquire-release semantics. +* IncSleepers: The number of sleeping threads is incremented, which is atomic exchange to C. +* SleepFence: A sequentially consistent fence is executed. * ReadJob: We look to see if the queue is empty, which is a read of J with acquire semantics. -Both IncJec and IncSleepers modify the same memory location, and hence they must be fully ordered. +Either PushFence or SleepFence must come first: -* If IncSleepers came first, there is no problem, because the injecting thread - knows that everyone is asleep and it will wake up a thread. -* If IncJec came first, then it "synchronizes with" IncSleepers. - * Therefore, PushJob "happens before" ReadJob, and so the write will be visible during - this final check, and the thread will not go to sleep. +* If PushFence comes first, then PushJob must be visible to ReadJob. +* If SleepFence comes first, then IncSleepers is visible to ReadSleepers. \ No newline at end of file diff --git a/rayon-core/src/sleep/counters.rs b/rayon-core/src/sleep/counters.rs index 5c94bc6b1..1d4d8d1d3 100644 --- a/rayon-core/src/sleep/counters.rs +++ b/rayon-core/src/sleep/counters.rs @@ -32,6 +32,21 @@ impl JobsEventCounter { pub(super) fn as_usize(self) -> usize { self.0 } + + /// The JEC "is sleepy" if the last thread to increment it was in the + /// process of becoming sleepy. This is indicated by its value being *even*. + /// When new jobs are posted, they check if the JEC is sleepy, and if so + /// they incremented it. + pub(super) fn is_sleepy(self) -> bool { + (self.as_usize() & 0) == 0 + } + + /// The JEC "is active" if the last thread to increment it was posting new + /// work. This is indicated by its value being *odd*. When threads get + /// sleepy, they will check if the JEC is active, and increment it. + pub(super) fn is_active(self) -> bool { + !self.is_sleepy() + } } /// Number of bits used for the thread counters. @@ -97,16 +112,25 @@ impl AtomicCounters { self.value.fetch_add(ONE_INACTIVE, Ordering::SeqCst); } - /// Attempts to increment the jobs event counter by one, returning true - /// if it succeeded. This can be used for two purposes: - /// - /// * If a thread is getting sleepy, and the JEC is even, then it will attempt - /// to increment to an odd value. - /// * If a thread is publishing work, and the JEC is odd, then it will attempt - /// to increment to an event value. - #[inline] - pub(super) fn increment_and_read_jobs_event_counter(&self) -> Counters { - Counters::new(self.value.fetch_add(ONE_JEC, Ordering::SeqCst)) + /// Increments the jobs event counter if `increment_when`, when applied to + /// the current value, is true. Used to toggle the JEC from even (sleepy) to + /// odd (active) or vice versa. Returns the final value of the counters, for + /// which `increment_when` is guaranteed to return false. + pub(super) fn increment_jobs_event_counter_if( + &self, + increment_when: impl Fn(JobsEventCounter) -> bool, + ) -> Counters { + loop { + let old_value = self.load(Ordering::SeqCst); + if increment_when(old_value.jobs_counter()) { + let new_value = old_value.plus(ONE_JEC); + if self.try_exchange(old_value, new_value, Ordering::SeqCst) { + return new_value; + } + } else { + return old_value; + } + } } /// Subtracts an inactive thread. This cannot fail. It is invoked @@ -193,6 +217,12 @@ impl Counters { Counters { word } } + fn plus(self, word: usize) -> Counters { + Counters { + word: self.word + word, + } + } + #[inline] pub(super) fn jobs_counter(self) -> JobsEventCounter { JobsEventCounter(select_jec(self.word)) diff --git a/rayon-core/src/sleep/mod.rs b/rayon-core/src/sleep/mod.rs index 4db30c58a..2aa262c51 100644 --- a/rayon-core/src/sleep/mod.rs +++ b/rayon-core/src/sleep/mod.rs @@ -123,7 +123,9 @@ impl Sleep { #[cold] fn announce_sleepy(&self, worker_index: usize) -> JobsEventCounter { - let counters = self.counters.load(Ordering::SeqCst); + let counters = self + .counters + .increment_jobs_event_counter_if(JobsEventCounter::is_active); let jobs_counter = counters.jobs_counter(); self.logger.log(|| ThreadSleepy { worker: worker_index, @@ -170,6 +172,7 @@ impl Sleep { let counters = self.counters.load(Ordering::SeqCst); // Check if the JEC has changed since we got sleepy. + debug_assert!(idle_state.jobs_counter.is_sleepy()); if counters.jobs_counter() != idle_state.jobs_counter { // JEC has changed, so a new job was posted, but for some reason // we didn't see it. We should return to just before the SLEEPY @@ -203,6 +206,7 @@ impl Sleep { // - an external job is being injected while we are sleepy // - that job triggers the rollover over the JEC such that we don't see it // - we are the last active worker thread + std::sync::atomic::fence(Ordering::SeqCst); if has_injected_jobs() { // If we see an externally injected job, then we have to 'wake // ourselves up'. (Ordinarily, `sub_sleeping_thread` is invoked by @@ -260,6 +264,11 @@ impl Sleep { num_jobs: u32, queue_was_empty: bool, ) { + // This fence is needed to guarantee that threads + // as they are about to fall asleep, observe any + // new jobs that may have been injected. + std::sync::atomic::fence(Ordering::SeqCst); + self.new_jobs(source_worker_index, num_jobs, queue_was_empty) } @@ -294,8 +303,9 @@ impl Sleep { // Read the counters and -- if sleepy workers have announced themselves // -- announce that there is now work available. The final value of `counters` // with which we exit the loop thus corresponds to a state when - let counters = self.counters.increment_and_read_jobs_event_counter(); - + let counters = self + .counters + .increment_jobs_event_counter_if(JobsEventCounter::is_sleepy); let num_awake_but_idle = counters.awake_but_idle_threads(); let num_sleepers = counters.sleeping_threads(); From c3a5304df899a0b962dc9b2dadf6d6887c5e8bf1 Mon Sep 17 00:00:00 2001 From: Josh Stone Date: Fri, 15 May 2020 12:28:46 -0700 Subject: [PATCH 31/33] Correct the bit test in JobsEventCounter::is_sleepy --- rayon-core/src/sleep/counters.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rayon-core/src/sleep/counters.rs b/rayon-core/src/sleep/counters.rs index 1d4d8d1d3..9064c6757 100644 --- a/rayon-core/src/sleep/counters.rs +++ b/rayon-core/src/sleep/counters.rs @@ -38,7 +38,7 @@ impl JobsEventCounter { /// When new jobs are posted, they check if the JEC is sleepy, and if so /// they incremented it. pub(super) fn is_sleepy(self) -> bool { - (self.as_usize() & 0) == 0 + (self.as_usize() & 1) == 0 } /// The JEC "is active" if the last thread to increment it was posting new From 96ba9ef188b4bd5f20b162fb9b861df134e27e9a Mon Sep 17 00:00:00 2001 From: Josh Stone Date: Fri, 15 May 2020 12:47:32 -0700 Subject: [PATCH 32/33] inline more Counter methods --- rayon-core/src/sleep/counters.rs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/rayon-core/src/sleep/counters.rs b/rayon-core/src/sleep/counters.rs index 9064c6757..626e24f1e 100644 --- a/rayon-core/src/sleep/counters.rs +++ b/rayon-core/src/sleep/counters.rs @@ -29,6 +29,7 @@ pub(super) struct JobsEventCounter(usize); impl JobsEventCounter { pub(super) const DUMMY: JobsEventCounter = JobsEventCounter(std::usize::MAX); + #[inline] pub(super) fn as_usize(self) -> usize { self.0 } @@ -37,6 +38,7 @@ impl JobsEventCounter { /// process of becoming sleepy. This is indicated by its value being *even*. /// When new jobs are posted, they check if the JEC is sleepy, and if so /// they incremented it. + #[inline] pub(super) fn is_sleepy(self) -> bool { (self.as_usize() & 1) == 0 } @@ -44,6 +46,7 @@ impl JobsEventCounter { /// The JEC "is active" if the last thread to increment it was posting new /// work. This is indicated by its value being *odd*. When threads get /// sleepy, they will check if the JEC is active, and increment it. + #[inline] pub(super) fn is_active(self) -> bool { !self.is_sleepy() } @@ -217,6 +220,7 @@ impl Counters { Counters { word } } + #[inline] fn plus(self, word: usize) -> Counters { Counters { word: self.word + word, From ed6a5f75c4dc1cb0ac4f59f6f01b38f8a695a777 Mon Sep 17 00:00:00 2001 From: Josh Stone Date: Thu, 13 Aug 2020 12:43:00 -0700 Subject: [PATCH 33/33] Update ci/compat-Cargo.lock --- ci/compat-Cargo.lock | 314 ++++++++++++++++++++++--------------------- 1 file changed, 162 insertions(+), 152 deletions(-) diff --git a/ci/compat-Cargo.lock b/ci/compat-Cargo.lock index 9d655ddc3..2e98c07b0 100644 --- a/ci/compat-Cargo.lock +++ b/ci/compat-Cargo.lock @@ -2,20 +2,20 @@ # It is not intended for manual editing. [[package]] name = "addr2line" -version = "0.12.1" +version = "0.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "gimli 0.21.0 (registry+https://github.com/rust-lang/crates.io-index)", + "gimli 0.22.0 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] -name = "adler32" -version = "1.1.0" +name = "adler" +version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] name = "aho-corasick" -version = "0.7.10" +version = "0.7.13" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "memchr 2.3.3 (registry+https://github.com/rust-lang/crates.io-index)", @@ -64,13 +64,13 @@ source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] name = "backtrace" -version = "0.3.49" +version = "0.3.50" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "addr2line 0.12.1 (registry+https://github.com/rust-lang/crates.io-index)", + "addr2line 0.13.0 (registry+https://github.com/rust-lang/crates.io-index)", "cfg-if 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)", - "libc 0.2.71 (registry+https://github.com/rust-lang/crates.io-index)", - "miniz_oxide 0.3.7 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.74 (registry+https://github.com/rust-lang/crates.io-index)", + "miniz_oxide 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", "object 0.20.0 (registry+https://github.com/rust-lang/crates.io-index)", "rustc-demangle 0.1.16 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -102,7 +102,7 @@ dependencies = [ [[package]] name = "cc" -version = "1.0.54" +version = "1.0.58" source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] @@ -115,7 +115,7 @@ name = "cgl" version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "libc 0.2.71 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.74 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -138,15 +138,15 @@ dependencies = [ [[package]] name = "cocoa" -version = "0.20.1" +version = "0.20.2" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "bitflags 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)", "block 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)", "core-foundation 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)", - "core-graphics 0.19.0 (registry+https://github.com/rust-lang/crates.io-index)", + "core-graphics 0.19.2 (registry+https://github.com/rust-lang/crates.io-index)", "foreign-types 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)", - "libc 0.2.71 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.74 (registry+https://github.com/rust-lang/crates.io-index)", "objc 0.2.7 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -156,7 +156,7 @@ version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "core-foundation-sys 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)", - "libc 0.2.71 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.74 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -166,13 +166,13 @@ source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] name = "core-graphics" -version = "0.19.0" +version = "0.19.2" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "bitflags 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)", "core-foundation 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)", "foreign-types 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)", - "libc 0.2.71 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.74 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -182,8 +182,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "cfg-if 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)", "core-foundation-sys 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)", - "core-graphics 0.19.0 (registry+https://github.com/rust-lang/crates.io-index)", - "libc 0.2.71 (registry+https://github.com/rust-lang/crates.io-index)", + "core-graphics 0.19.2 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.74 (registry+https://github.com/rust-lang/crates.io-index)", "objc 0.2.7 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -216,7 +216,7 @@ dependencies = [ "crossbeam-utils 0.7.2 (registry+https://github.com/rust-lang/crates.io-index)", "lazy_static 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)", "maybe-uninit 2.0.0 (registry+https://github.com/rust-lang/crates.io-index)", - "memoffset 0.5.4 (registry+https://github.com/rust-lang/crates.io-index)", + "memoffset 0.5.5 (registry+https://github.com/rust-lang/crates.io-index)", "scopeguard 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -235,9 +235,19 @@ name = "crossbeam-utils" version = "0.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "proc-macro2 1.0.18 (registry+https://github.com/rust-lang/crates.io-index)", + "autocfg 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)", + "cfg-if 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)", + "lazy_static 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "derivative" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "proc-macro2 1.0.19 (registry+https://github.com/rust-lang/crates.io-index)", "quote 1.0.7 (registry+https://github.com/rust-lang/crates.io-index)", - "syn 1.0.31 (registry+https://github.com/rust-lang/crates.io-index)", + "syn 1.0.38 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -265,18 +275,18 @@ source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "lazy_static 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)", "regex 1.3.9 (registry+https://github.com/rust-lang/crates.io-index)", - "serde 1.0.112 (registry+https://github.com/rust-lang/crates.io-index)", + "serde 1.0.115 (registry+https://github.com/rust-lang/crates.io-index)", "strsim 0.9.3 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] name = "downcast-rs" -version = "1.1.1" +version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] name = "either" -version = "1.5.3" +version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] @@ -327,13 +337,13 @@ version = "0.1.14" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "cfg-if 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)", - "libc 0.2.71 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.74 (registry+https://github.com/rust-lang/crates.io-index)", "wasi 0.9.0+wasi-snapshot-preview1 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] name = "gimli" -version = "0.21.0" +version = "0.22.0" source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] @@ -342,7 +352,7 @@ version = "0.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "khronos_api 3.1.0 (registry+https://github.com/rust-lang/crates.io-index)", - "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)", + "log 0.4.11 (registry+https://github.com/rust-lang/crates.io-index)", "xml-rs 0.8.3 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -352,7 +362,7 @@ version = "0.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "khronos_api 3.1.0 (registry+https://github.com/rust-lang/crates.io-index)", - "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)", + "log 0.4.11 (registry+https://github.com/rust-lang/crates.io-index)", "xml-rs 0.8.3 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -361,13 +371,13 @@ name = "glium" version = "0.27.0" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "backtrace 0.3.49 (registry+https://github.com/rust-lang/crates.io-index)", + "backtrace 0.3.50 (registry+https://github.com/rust-lang/crates.io-index)", "fnv 1.0.7 (registry+https://github.com/rust-lang/crates.io-index)", "gl_generator 0.14.0 (registry+https://github.com/rust-lang/crates.io-index)", "glutin 0.24.1 (registry+https://github.com/rust-lang/crates.io-index)", "lazy_static 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)", - "memoffset 0.5.4 (registry+https://github.com/rust-lang/crates.io-index)", - "smallvec 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)", + "memoffset 0.5.5 (registry+https://github.com/rust-lang/crates.io-index)", + "smallvec 1.4.2 (registry+https://github.com/rust-lang/crates.io-index)", "takeable-option 0.5.0 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -378,9 +388,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "android_glue 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)", "cgl 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)", - "cocoa 0.20.1 (registry+https://github.com/rust-lang/crates.io-index)", + "cocoa 0.20.2 (registry+https://github.com/rust-lang/crates.io-index)", "core-foundation 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)", - "core-graphics 0.19.0 (registry+https://github.com/rust-lang/crates.io-index)", + "core-graphics 0.19.2 (registry+https://github.com/rust-lang/crates.io-index)", "glutin_egl_sys 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)", "glutin_emscripten_sys 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", "glutin_gles2_sys 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)", @@ -388,12 +398,12 @@ dependencies = [ "glutin_wgl_sys 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)", "lazy_static 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)", "libloading 0.5.2 (registry+https://github.com/rust-lang/crates.io-index)", - "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)", + "log 0.4.11 (registry+https://github.com/rust-lang/crates.io-index)", "objc 0.2.7 (registry+https://github.com/rust-lang/crates.io-index)", "osmesa-sys 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", "parking_lot 0.10.2 (registry+https://github.com/rust-lang/crates.io-index)", "wayland-client 0.23.6 (registry+https://github.com/rust-lang/crates.io-index)", - "winapi 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)", + "winapi 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)", "winit 0.22.2 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -403,7 +413,7 @@ version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "gl_generator 0.13.1 (registry+https://github.com/rust-lang/crates.io-index)", - "winapi 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)", + "winapi 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -439,15 +449,15 @@ dependencies = [ [[package]] name = "hermit-abi" -version = "0.1.14" +version = "0.1.15" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "libc 0.2.71 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.74 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] name = "instant" -version = "0.1.4" +version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] @@ -455,7 +465,7 @@ name = "iovec" version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "libc 0.2.71 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.74 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -484,12 +494,12 @@ source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] name = "lazycell" -version = "1.2.1" +version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] name = "libc" -version = "0.2.71" +version = "0.2.74" source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] @@ -497,8 +507,8 @@ name = "libloading" version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "cc 1.0.54 (registry+https://github.com/rust-lang/crates.io-index)", - "winapi 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)", + "cc 1.0.58 (registry+https://github.com/rust-lang/crates.io-index)", + "winapi 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -506,7 +516,7 @@ name = "libloading" version = "0.6.2" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "winapi 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)", + "winapi 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -527,7 +537,7 @@ dependencies = [ [[package]] name = "log" -version = "0.4.8" +version = "0.4.11" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "cfg-if 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)", @@ -538,7 +548,7 @@ name = "malloc_buf" version = "0.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "libc 0.2.71 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.74 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -556,13 +566,13 @@ name = "memmap" version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "libc 0.2.71 (registry+https://github.com/rust-lang/crates.io-index)", - "winapi 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.74 (registry+https://github.com/rust-lang/crates.io-index)", + "winapi 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] name = "memoffset" -version = "0.5.4" +version = "0.5.5" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "autocfg 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)", @@ -570,10 +580,10 @@ dependencies = [ [[package]] name = "miniz_oxide" -version = "0.3.7" +version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "adler32 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "adler 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -586,8 +596,8 @@ dependencies = [ "fuchsia-zircon-sys 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)", "iovec 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)", "kernel32-sys 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)", - "libc 0.2.71 (registry+https://github.com/rust-lang/crates.io-index)", - "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.74 (registry+https://github.com/rust-lang/crates.io-index)", + "log 0.4.11 (registry+https://github.com/rust-lang/crates.io-index)", "miow 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", "net2 0.2.34 (registry+https://github.com/rust-lang/crates.io-index)", "slab 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)", @@ -599,8 +609,8 @@ name = "mio-extras" version = "2.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "lazycell 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)", - "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)", + "lazycell 1.3.0 (registry+https://github.com/rust-lang/crates.io-index)", + "log 0.4.11 (registry+https://github.com/rust-lang/crates.io-index)", "mio 0.6.22 (registry+https://github.com/rust-lang/crates.io-index)", "slab 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -633,8 +643,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "android_log-sys 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", "lazy_static 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)", - "libc 0.2.71 (registry+https://github.com/rust-lang/crates.io-index)", - "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.74 (registry+https://github.com/rust-lang/crates.io-index)", + "log 0.4.11 (registry+https://github.com/rust-lang/crates.io-index)", "ndk 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", "ndk-sys 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -650,8 +660,8 @@ version = "0.2.34" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "cfg-if 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)", - "libc 0.2.71 (registry+https://github.com/rust-lang/crates.io-index)", - "winapi 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.74 (registry+https://github.com/rust-lang/crates.io-index)", + "winapi 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -660,9 +670,9 @@ version = "0.14.1" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "bitflags 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)", - "cc 1.0.54 (registry+https://github.com/rust-lang/crates.io-index)", + "cc 1.0.58 (registry+https://github.com/rust-lang/crates.io-index)", "cfg-if 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)", - "libc 0.2.71 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.74 (registry+https://github.com/rust-lang/crates.io-index)", "void 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -741,8 +751,8 @@ name = "num_cpus" version = "1.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "hermit-abi 0.1.14 (registry+https://github.com/rust-lang/crates.io-index)", - "libc 0.2.71 (registry+https://github.com/rust-lang/crates.io-index)", + "hermit-abi 0.1.15 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.74 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -759,10 +769,10 @@ name = "num_enum_derive" version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "proc-macro-crate 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)", - "proc-macro2 1.0.18 (registry+https://github.com/rust-lang/crates.io-index)", + "proc-macro-crate 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)", + "proc-macro2 1.0.19 (registry+https://github.com/rust-lang/crates.io-index)", "quote 1.0.7 (registry+https://github.com/rust-lang/crates.io-index)", - "syn 1.0.31 (registry+https://github.com/rust-lang/crates.io-index)", + "syn 1.0.38 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -780,7 +790,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] name = "ordered-float" -version = "1.0.2" +version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "num-traits 0.2.12 (registry+https://github.com/rust-lang/crates.io-index)", @@ -810,10 +820,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "cfg-if 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)", "cloudabi 0.0.3 (registry+https://github.com/rust-lang/crates.io-index)", - "libc 0.2.71 (registry+https://github.com/rust-lang/crates.io-index)", - "redox_syscall 0.1.56 (registry+https://github.com/rust-lang/crates.io-index)", - "smallvec 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)", - "winapi 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.74 (registry+https://github.com/rust-lang/crates.io-index)", + "redox_syscall 0.1.57 (registry+https://github.com/rust-lang/crates.io-index)", + "smallvec 1.4.2 (registry+https://github.com/rust-lang/crates.io-index)", + "winapi 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -823,7 +833,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] name = "pkg-config" -version = "0.3.17" +version = "0.3.18" source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] @@ -833,7 +843,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] name = "proc-macro-crate" -version = "0.1.4" +version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "toml 0.5.6 (registry+https://github.com/rust-lang/crates.io-index)", @@ -849,10 +859,10 @@ dependencies = [ [[package]] name = "proc-macro2" -version = "1.0.18" +version = "1.0.19" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "unicode-xid 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", + "unicode-xid 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -868,7 +878,7 @@ name = "quote" version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "proc-macro2 1.0.18 (registry+https://github.com/rust-lang/crates.io-index)", + "proc-macro2 1.0.19 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -877,7 +887,7 @@ version = "0.6.5" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "autocfg 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)", - "libc 0.2.71 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.74 (registry+https://github.com/rust-lang/crates.io-index)", "rand_chacha 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", "rand_core 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)", "rand_hc 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", @@ -886,7 +896,7 @@ dependencies = [ "rand_os 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)", "rand_pcg 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", "rand_xorshift 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", - "winapi 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)", + "winapi 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -895,7 +905,7 @@ version = "0.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "getrandom 0.1.14 (registry+https://github.com/rust-lang/crates.io-index)", - "libc 0.2.71 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.74 (registry+https://github.com/rust-lang/crates.io-index)", "rand_chacha 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)", "rand_core 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)", "rand_hc 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", @@ -969,9 +979,9 @@ name = "rand_jitter" version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "libc 0.2.71 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.74 (registry+https://github.com/rust-lang/crates.io-index)", "rand_core 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)", - "winapi 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)", + "winapi 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -981,10 +991,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "cloudabi 0.0.3 (registry+https://github.com/rust-lang/crates.io-index)", "fuchsia-cprng 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", - "libc 0.2.71 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.74 (registry+https://github.com/rust-lang/crates.io-index)", "rand_core 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)", "rdrand 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", - "winapi 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)", + "winapi 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -1017,7 +1027,7 @@ name = "raw-window-handle" version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "libc 0.2.71 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.74 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -1027,12 +1037,12 @@ dependencies = [ "autocfg 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)", "crossbeam-deque 0.7.3 (registry+https://github.com/rust-lang/crates.io-index)", "docopt 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)", - "either 1.5.3 (registry+https://github.com/rust-lang/crates.io-index)", + "either 1.6.0 (registry+https://github.com/rust-lang/crates.io-index)", "lazy_static 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)", "rand 0.7.3 (registry+https://github.com/rust-lang/crates.io-index)", "rand_xorshift 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", "rayon-core 1.7.1", - "serde 1.0.112 (registry+https://github.com/rust-lang/crates.io-index)", + "serde 1.0.115 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -1044,7 +1054,7 @@ dependencies = [ "crossbeam-queue 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)", "crossbeam-utils 0.7.2 (registry+https://github.com/rust-lang/crates.io-index)", "lazy_static 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)", - "libc 0.2.71 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.74 (registry+https://github.com/rust-lang/crates.io-index)", "num_cpus 1.13.0 (registry+https://github.com/rust-lang/crates.io-index)", "rand 0.7.3 (registry+https://github.com/rust-lang/crates.io-index)", "rand_xorshift 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", @@ -1061,14 +1071,14 @@ dependencies = [ "fixedbitset 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)", "glium 0.27.0 (registry+https://github.com/rust-lang/crates.io-index)", "lazy_static 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)", - "libc 0.2.71 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.74 (registry+https://github.com/rust-lang/crates.io-index)", "num 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", "rand 0.7.3 (registry+https://github.com/rust-lang/crates.io-index)", "rand_xorshift 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", "rayon 1.3.1", "regex 1.3.9 (registry+https://github.com/rust-lang/crates.io-index)", - "serde 1.0.112 (registry+https://github.com/rust-lang/crates.io-index)", - "winapi 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)", + "serde 1.0.115 (registry+https://github.com/rust-lang/crates.io-index)", + "winapi 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -1081,7 +1091,7 @@ dependencies = [ [[package]] name = "redox_syscall" -version = "0.1.56" +version = "0.1.57" source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] @@ -1089,7 +1099,7 @@ name = "regex" version = "1.3.9" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "aho-corasick 0.7.10 (registry+https://github.com/rust-lang/crates.io-index)", + "aho-corasick 0.7.13 (registry+https://github.com/rust-lang/crates.io-index)", "memchr 2.3.3 (registry+https://github.com/rust-lang/crates.io-index)", "regex-syntax 0.6.18 (registry+https://github.com/rust-lang/crates.io-index)", "thread_local 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)", @@ -1119,7 +1129,7 @@ version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "approx 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)", - "ordered-float 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)", + "ordered-float 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)", "stb_truetype 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -1143,20 +1153,20 @@ source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] name = "serde" -version = "1.0.112" +version = "1.0.115" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "serde_derive 1.0.112 (registry+https://github.com/rust-lang/crates.io-index)", + "serde_derive 1.0.115 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] name = "serde_derive" -version = "1.0.112" +version = "1.0.115" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "proc-macro2 1.0.18 (registry+https://github.com/rust-lang/crates.io-index)", + "proc-macro2 1.0.19 (registry+https://github.com/rust-lang/crates.io-index)", "quote 1.0.7 (registry+https://github.com/rust-lang/crates.io-index)", - "syn 1.0.31 (registry+https://github.com/rust-lang/crates.io-index)", + "syn 1.0.38 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -1165,7 +1175,7 @@ version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "lazy_static 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)", - "libc 0.2.71 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.74 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -1175,7 +1185,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] name = "smallvec" -version = "1.4.0" +version = "1.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] @@ -1208,12 +1218,12 @@ source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] name = "syn" -version = "1.0.31" +version = "1.0.38" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "proc-macro2 1.0.18 (registry+https://github.com/rust-lang/crates.io-index)", + "proc-macro2 1.0.19 (registry+https://github.com/rust-lang/crates.io-index)", "quote 1.0.7 (registry+https://github.com/rust-lang/crates.io-index)", - "unicode-xid 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", + "unicode-xid 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -1234,7 +1244,7 @@ name = "toml" version = "0.5.6" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "serde 1.0.112 (registry+https://github.com/rust-lang/crates.io-index)", + "serde 1.0.115 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -1244,7 +1254,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] name = "unicode-xid" -version = "0.2.0" +version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] @@ -1258,7 +1268,7 @@ version = "2.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "same-file 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)", - "winapi 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)", + "winapi 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)", "winapi-util 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -1274,8 +1284,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "bitflags 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)", "calloop 0.4.4 (registry+https://github.com/rust-lang/crates.io-index)", - "downcast-rs 1.1.1 (registry+https://github.com/rust-lang/crates.io-index)", - "libc 0.2.71 (registry+https://github.com/rust-lang/crates.io-index)", + "downcast-rs 1.2.0 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.74 (registry+https://github.com/rust-lang/crates.io-index)", "mio 0.6.22 (registry+https://github.com/rust-lang/crates.io-index)", "nix 0.14.1 (registry+https://github.com/rust-lang/crates.io-index)", "wayland-commons 0.23.6 (registry+https://github.com/rust-lang/crates.io-index)", @@ -1329,7 +1339,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] name = "winapi" -version = "0.3.8" +version = "0.3.9" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "winapi-i686-pc-windows-gnu 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", @@ -1351,7 +1361,7 @@ name = "winapi-util" version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "winapi 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)", + "winapi 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -1365,15 +1375,15 @@ version = "0.22.2" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "bitflags 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)", - "cocoa 0.20.1 (registry+https://github.com/rust-lang/crates.io-index)", + "cocoa 0.20.2 (registry+https://github.com/rust-lang/crates.io-index)", "core-foundation 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)", - "core-graphics 0.19.0 (registry+https://github.com/rust-lang/crates.io-index)", + "core-graphics 0.19.2 (registry+https://github.com/rust-lang/crates.io-index)", "core-video-sys 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)", "dispatch 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", - "instant 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)", + "instant 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)", "lazy_static 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)", - "libc 0.2.71 (registry+https://github.com/rust-lang/crates.io-index)", - "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.74 (registry+https://github.com/rust-lang/crates.io-index)", + "log 0.4.11 (registry+https://github.com/rust-lang/crates.io-index)", "mio 0.6.22 (registry+https://github.com/rust-lang/crates.io-index)", "mio-extras 2.0.6 (registry+https://github.com/rust-lang/crates.io-index)", "ndk 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", @@ -1385,7 +1395,7 @@ dependencies = [ "raw-window-handle 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)", "smithay-client-toolkit 0.6.6 (registry+https://github.com/rust-lang/crates.io-index)", "wayland-client 0.23.6 (registry+https://github.com/rust-lang/crates.io-index)", - "winapi 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)", + "winapi 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)", "x11-dl 2.18.5 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -1404,9 +1414,9 @@ version = "2.18.5" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "lazy_static 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)", - "libc 0.2.71 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.74 (registry+https://github.com/rust-lang/crates.io-index)", "maybe-uninit 2.0.0 (registry+https://github.com/rust-lang/crates.io-index)", - "pkg-config 0.3.17 (registry+https://github.com/rust-lang/crates.io-index)", + "pkg-config 0.3.18 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -1420,42 +1430,42 @@ version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" [metadata] -"checksum addr2line 0.12.1 (registry+https://github.com/rust-lang/crates.io-index)" = "a49806b9dadc843c61e7c97e72490ad7f7220ae249012fbda9ad0609457c0543" -"checksum adler32 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "567b077b825e468cc974f0020d4082ee6e03132512f207ef1a02fd5d00d1f32d" -"checksum aho-corasick 0.7.10 (registry+https://github.com/rust-lang/crates.io-index)" = "8716408b8bc624ed7f65d223ddb9ac2d044c0547b6fa4b0d554f3a9540496ada" +"checksum addr2line 0.13.0 (registry+https://github.com/rust-lang/crates.io-index)" = "1b6a2d3371669ab3ca9797670853d61402b03d0b4b9ebf33d677dfa720203072" +"checksum adler 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)" = "ee2a4ec343196209d6594e19543ae87a39f96d5534d7174822a3ad825dd6ed7e" +"checksum aho-corasick 0.7.13 (registry+https://github.com/rust-lang/crates.io-index)" = "043164d8ba5c4c3035fec9bbee8647c0261d788f3474306f93bb65901cae0e86" "checksum andrew 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "9b7f09f89872c2b6b29e319377b1fbe91c6f5947df19a25596e121cf19a7b35e" "checksum android_glue 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)" = "000444226fcff248f2bc4c7625be32c63caccfecc2723a2b9f78a7487a49c407" "checksum android_log-sys 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)" = "b8052e2d8aabbb8d556d6abbcce2a22b9590996c5f849b9c7ce4544a2e3b984e" "checksum approx 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)" = "f0e60b75072ecd4168020818c0107f2857bb6c4e64252d8d3983f6263b40a5c3" "checksum autocfg 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)" = "1d49d90015b3c36167a20fe2810c5cd875ad504b39cff3d4eae7977e6b7c1cb2" "checksum autocfg 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "f8aac770f1885fd7e387acedd76065302551364496e46b3dd00860b2f8359b9d" -"checksum backtrace 0.3.49 (registry+https://github.com/rust-lang/crates.io-index)" = "05100821de9e028f12ae3d189176b41ee198341eb8f369956407fea2f5cc666c" +"checksum backtrace 0.3.50 (registry+https://github.com/rust-lang/crates.io-index)" = "46254cf2fdcdf1badb5934448c1bcbe046a56537b3987d96c51a7afc5d03f293" "checksum bitflags 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "cf1de2fe8c75bc145a2f577add951f8134889b4795d47466a54a5c846d691693" "checksum block 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)" = "0d8c1fef690941d3e7788d328517591fecc684c084084702d6ff1641e993699a" "checksum byteorder 1.3.4 (registry+https://github.com/rust-lang/crates.io-index)" = "08c48aae112d48ed9f069b33538ea9e3e90aa263cfa3d1c24309612b1f7472de" "checksum calloop 0.4.4 (registry+https://github.com/rust-lang/crates.io-index)" = "7aa2097be53a00de9e8fc349fea6d76221f398f5c4fa550d420669906962d160" -"checksum cc 1.0.54 (registry+https://github.com/rust-lang/crates.io-index)" = "7bbb73db36c1246e9034e307d0fba23f9a2e251faa47ade70c1bd252220c8311" +"checksum cc 1.0.58 (registry+https://github.com/rust-lang/crates.io-index)" = "f9a06fb2e53271d7c279ec1efea6ab691c35a2ae67ec0d91d7acec0caf13b518" "checksum cfg-if 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)" = "4785bdd1c96b2a846b2bd7cc02e86b6b3dbf14e7e53446c4f54c92a361040822" "checksum cgl 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)" = "0ced0551234e87afee12411d535648dd89d2e7f34c78b753395567aff3d447ff" "checksum cgmath 0.17.0 (registry+https://github.com/rust-lang/crates.io-index)" = "283944cdecc44bf0b8dd010ec9af888d3b4f142844fdbe026c20ef68148d6fe7" "checksum cloudabi 0.0.3 (registry+https://github.com/rust-lang/crates.io-index)" = "ddfc5b9aa5d4507acaf872de71051dfd0e309860e88966e1051e462a077aac4f" -"checksum cocoa 0.19.1 (registry+https://github.com/rust-lang/crates.io-index)" = "f29f7768b2d1be17b96158e3285951d366b40211320fb30826a76cb7a0da6400" -"checksum core-foundation 0.6.4 (registry+https://github.com/rust-lang/crates.io-index)" = "25b9e03f145fd4f2bf705e07b900cd41fc636598fe5dc452fd0db1441c3f496d" -"checksum core-foundation-sys 0.6.2 (registry+https://github.com/rust-lang/crates.io-index)" = "e7ca8a5221364ef15ce201e8ed2f609fc312682a8f4e0e3d4aa5879764e0fa3b" -"checksum core-graphics 0.17.3 (registry+https://github.com/rust-lang/crates.io-index)" = "56790968ab1c8a1202a102e6de05fc6e1ec87da99e4e93e9a7d13efbfc1e95a9" -"checksum core-video-sys 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)" = "8dc065219542086f72d1e9f7aadbbab0989e980263695d129d502082d063a9d0" +"checksum cocoa 0.20.2 (registry+https://github.com/rust-lang/crates.io-index)" = "0c49e86fc36d5704151f5996b7b3795385f50ce09e3be0f47a0cfde869681cf8" +"checksum core-foundation 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)" = "57d24c7a13c43e870e37c1556b74555437870a04514f7685f5b354e090567171" +"checksum core-foundation-sys 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)" = "b3a71ab494c0b5b860bdc8407ae08978052417070c2ced38573a9157ad75b8ac" +"checksum core-graphics 0.19.2 (registry+https://github.com/rust-lang/crates.io-index)" = "b3889374e6ea6ab25dba90bb5d96202f61108058361f6dc72e8b03e6f8bbe923" +"checksum core-video-sys 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)" = "34ecad23610ad9757664d644e369246edde1803fcb43ed72876565098a5d3828" "checksum crossbeam-channel 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)" = "cced8691919c02aac3cb0a1bc2e9b73d89e832bf9a06fc579d4e71b68a2da061" "checksum crossbeam-deque 0.7.3 (registry+https://github.com/rust-lang/crates.io-index)" = "9f02af974daeee82218205558e51ec8768b48cf524bd01d550abe5573a608285" "checksum crossbeam-epoch 0.8.2 (registry+https://github.com/rust-lang/crates.io-index)" = "058ed274caafc1f60c4997b5fc07bf7dc7cca454af7c6e81edffe5f33f70dace" -"checksum crossbeam-queue 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "c695eeca1e7173472a32221542ae469b3e9aac3a4fc81f7696bcad82029493db" +"checksum crossbeam-queue 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)" = "774ba60a54c213d409d5353bda12d49cd68d14e45036a285234c8d6f91f92570" "checksum crossbeam-utils 0.7.2 (registry+https://github.com/rust-lang/crates.io-index)" = "c3c7c73a2d1e9fc0886a08b93e98eb643461230d5f1925e4036204d5f2e261a8" "checksum derivative 2.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "cb582b60359da160a9477ee80f15c8d784c477e69c217ef2cdd4169c24ea380f" "checksum dispatch 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "bd0c93bb4b0c6d9b77f4435b0ae98c24d17f1c45b2ff844c6151a07256ca923b" "checksum dlib 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)" = "b11f15d1e3268f140f68d390637d5e76d849782d971ae7063e0da69fe9709a76" "checksum doc-comment 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)" = "fea41bba32d969b513997752735605054bc0dfa92b4c56bf1189f2e174be7a10" "checksum docopt 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "7f525a586d310c87df72ebcd98009e57f1cc030c8c268305287a476beb653969" -"checksum downcast-rs 1.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "52ba6eb47c2131e784a38b726eb54c1e1484904f013e576a25354d0124161af6" -"checksum either 1.5.3 (registry+https://github.com/rust-lang/crates.io-index)" = "bb1f6b1ce1c140482ea30ddd3335fc0024ac7ee112895426e0a629a6c20adfe3" +"checksum downcast-rs 1.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "9ea835d29036a4087793836fa931b08837ad5e957da9e23886b29586fb9b6650" +"checksum either 1.6.0 (registry+https://github.com/rust-lang/crates.io-index)" = "cd56b59865bce947ac5958779cfa508f6c3b9497cc762b7e24a12d11ccde2c4f" "checksum fixedbitset 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "2fc4fcacf5cd3681968f6524ea159383132937739c6c40dabab9e37ed515911b" "checksum fnv 1.0.7 (registry+https://github.com/rust-lang/crates.io-index)" = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" "checksum foreign-types 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)" = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1" @@ -1464,7 +1474,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" "checksum fuchsia-zircon 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)" = "2e9763c69ebaae630ba35f74888db465e49e259ba1bc0eda7d06f4a067615d82" "checksum fuchsia-zircon-sys 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)" = "3dcaa9ae7725d12cdb85b3ad99a434db70b468c09ded17e012d86b5c1010f7a7" "checksum getrandom 0.1.14 (registry+https://github.com/rust-lang/crates.io-index)" = "7abc8dd8451921606d809ba32e95b6111925cd2906060d2dcc29c070220503eb" -"checksum gimli 0.21.0 (registry+https://github.com/rust-lang/crates.io-index)" = "bcc8e0c9bce37868955864dbecd2b1ab2bdf967e6f28066d65aaac620444b65c" +"checksum gimli 0.22.0 (registry+https://github.com/rust-lang/crates.io-index)" = "aaf91faf136cb47367fa430cd46e37a788775e7fa104f8b4bcb3861dc389b724" "checksum gl_generator 0.13.1 (registry+https://github.com/rust-lang/crates.io-index)" = "ca98bbde17256e02d17336a6bdb5a50f7d0ccacee502e191d3e3d0ec2f96f84a" "checksum gl_generator 0.14.0 (registry+https://github.com/rust-lang/crates.io-index)" = "1a95dfc23a2b4a9a2f5ab41d194f8bfda3cabec42af4e39f08c339eb2a0c124d" "checksum glium 0.27.0 (registry+https://github.com/rust-lang/crates.io-index)" = "030bb23a12fac7e589b002c5e131e89348df88f91b56e3f3dbc4249527eeebf9" @@ -1474,26 +1484,26 @@ source = "registry+https://github.com/rust-lang/crates.io-index" "checksum glutin_gles2_sys 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)" = "07e853d96bebcb8e53e445225c3009758c6f5960d44f2543245f6f07b567dae0" "checksum glutin_glx_sys 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)" = "08c243de74d6cf5ea100c788826d2fb9319de315485dd4b310811a663b3809c3" "checksum glutin_wgl_sys 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)" = "a93dba7ee3a0feeac0f437141ff25e71ce2066bcf1a706acab1559ffff94eb6a" -"checksum hermit-abi 0.1.14 (registry+https://github.com/rust-lang/crates.io-index)" = "b9586eedd4ce6b3c498bc3b4dd92fc9f11166aa908a914071953768066c67909" -"checksum instant 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)" = "7777a24a1ce5de49fcdde84ec46efa487c3af49d5b6e6e0a50367cc5c1096182" +"checksum hermit-abi 0.1.15 (registry+https://github.com/rust-lang/crates.io-index)" = "3deed196b6e7f9e44a2ae8d94225d80302d81208b1bb673fd21fe634645c85a9" +"checksum instant 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)" = "5b141fdc7836c525d4d594027d318c84161ca17aaf8113ab1f81ab93ae897485" "checksum iovec 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)" = "b2b3ea6ff95e175473f8ffe6a7eb7c00d054240321b84c57051175fe3c1e075e" "checksum jni-sys 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "8eaf4bc02d17cbdd7ff4c7438cafcdf7fb9a4613313ad11b4f8fefe7d3fa0130" "checksum kernel32-sys 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "7507624b29483431c0ba2d82aece8ca6cdba9382bff4ddd0f7490560c056098d" "checksum khronos_api 3.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "e2db585e1d738fc771bf08a151420d3ed193d9d895a36df7f6f8a9456b911ddc" "checksum lazy_static 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" -"checksum lazycell 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "b294d6fa9ee409a054354afc4352b0b9ef7ca222c69b8812cbea9e7d2bf3783f" -"checksum libc 0.2.71 (registry+https://github.com/rust-lang/crates.io-index)" = "9457b06509d27052635f90d6466700c65095fdf75409b3fbdd903e988b886f49" +"checksum lazycell 1.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55" +"checksum libc 0.2.74 (registry+https://github.com/rust-lang/crates.io-index)" = "a2f02823cf78b754822df5f7f268fb59822e7296276d3e069d8e8cb26a14bd10" "checksum libloading 0.5.2 (registry+https://github.com/rust-lang/crates.io-index)" = "f2b111a074963af1d37a139918ac6d49ad1d0d5e47f72fd55388619691a7d753" "checksum libloading 0.6.2 (registry+https://github.com/rust-lang/crates.io-index)" = "2cadb8e769f070c45df05c78c7520eb4cd17061d4ab262e43cfc68b4d00ac71c" "checksum line_drawing 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)" = "5cc7ad3d82c845bdb5dde34ffdcc7a5fb4d2996e1e1ee0f19c33bc80e15196b9" "checksum lock_api 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)" = "c4da24a77a3d8a6d4862d95f72e6fdb9c09a643ecdb402d754004a557f2bec75" -"checksum log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)" = "14b6052be84e6b71ab17edffc2eeabf5c2c3ae1fdb464aae35ac50c67a44e1f7" +"checksum log 0.4.11 (registry+https://github.com/rust-lang/crates.io-index)" = "4fabed175da42fed1fa0746b0ea71f412aa9d35e76e95e59b192c64b9dc2bf8b" "checksum malloc_buf 0.0.6 (registry+https://github.com/rust-lang/crates.io-index)" = "62bb907fe88d54d8d9ce32a3cceab4218ed2f6b7d35617cafe9adf84e43919cb" "checksum maybe-uninit 2.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "60302e4db3a61da70c0cb7991976248362f30319e88850c487b9b95bbf059e00" "checksum memchr 2.3.3 (registry+https://github.com/rust-lang/crates.io-index)" = "3728d817d99e5ac407411fa471ff9800a778d88a24685968b36824eaf4bee400" "checksum memmap 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)" = "6585fd95e7bb50d6cc31e20d4cf9afb4e2ba16c5846fc76793f11218da9c475b" -"checksum memoffset 0.5.4 (registry+https://github.com/rust-lang/crates.io-index)" = "b4fc2c02a7e374099d4ee95a193111f72d2110197fe200272371758f6c3643d8" -"checksum miniz_oxide 0.3.7 (registry+https://github.com/rust-lang/crates.io-index)" = "791daaae1ed6889560f8c4359194f56648355540573244a5448a83ba1ecc7435" +"checksum memoffset 0.5.5 (registry+https://github.com/rust-lang/crates.io-index)" = "c198b026e1bbf08a937e94c6c60f9ec4a2267f5b0d2eec9c1b21b061ce2be55f" +"checksum miniz_oxide 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "be0f75932c1f6cfae3c04000e40114adf955636e19040f9c0a2c380702aa1c7f" "checksum mio 0.6.22 (registry+https://github.com/rust-lang/crates.io-index)" = "fce347092656428bc8eaf6201042cb551b8d67855af7374542a92a0fbfcac430" "checksum mio-extras 2.0.6 (registry+https://github.com/rust-lang/crates.io-index)" = "52403fe290012ce777c4626790c8951324a2b9e3316b3143779c72b029742f19" "checksum miow 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "8c1f2f3b1cf331de6896aabf6e9d55dca90356cc9960cca7eaaf408a355ae919" @@ -1514,16 +1524,16 @@ source = "registry+https://github.com/rust-lang/crates.io-index" "checksum num_enum_derive 0.4.3 (registry+https://github.com/rust-lang/crates.io-index)" = "ffa5a33ddddfee04c0283a7653987d634e880347e96b5b2ed64de07efb59db9d" "checksum objc 0.2.7 (registry+https://github.com/rust-lang/crates.io-index)" = "915b1b472bc21c53464d6c8461c9d3af805ba1ef837e1cac254428f4a77177b1" "checksum object 0.20.0 (registry+https://github.com/rust-lang/crates.io-index)" = "1ab52be62400ca80aa00285d25253d7f7c437b7375c4de678f5405d3afe82ca5" -"checksum ordered-float 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)" = "18869315e81473c951eb56ad5558bbc56978562d3ecfb87abb7a1e944cea4518" +"checksum ordered-float 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "3741934be594d77de1c8461ebcbbe866f585ea616a9753aa78f2bdc69f0e4579" "checksum osmesa-sys 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)" = "88cfece6e95d2e717e0872a7f53a8684712ad13822a7979bc760b9c77ec0013b" "checksum parking_lot 0.10.2 (registry+https://github.com/rust-lang/crates.io-index)" = "d3a704eb390aafdc107b0e392f56a82b668e3a71366993b5340f5833fd62505e" "checksum parking_lot_core 0.7.2 (registry+https://github.com/rust-lang/crates.io-index)" = "d58c7c768d4ba344e3e8d72518ac13e259d7c7ade24167003b8488e10b6740a3" "checksum percent-encoding 2.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "d4fd5641d01c8f18a23da7b6fe29298ff4b55afcccdf78973b24cf3175fee32e" -"checksum pkg-config 0.3.17 (registry+https://github.com/rust-lang/crates.io-index)" = "05da548ad6865900e60eaba7f589cc0783590a92e940c26953ff81ddbab2d677" +"checksum pkg-config 0.3.18 (registry+https://github.com/rust-lang/crates.io-index)" = "d36492546b6af1463394d46f0c834346f31548646f6ba10849802c9c9a27ac33" "checksum ppv-lite86 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)" = "237a5ed80e274dbc66f86bd59c1e25edc039660be53194b5fe0a482e0f2612ea" -"checksum proc-macro-crate 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)" = "e10d4b51f154c8a7fb96fd6dad097cb74b863943ec010ac94b9fd1be8861fe1e" +"checksum proc-macro-crate 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)" = "1d6ea3c4595b96363c13943497db34af4460fb474a95c43f4446ad341b8c9785" "checksum proc-macro2 0.4.30 (registry+https://github.com/rust-lang/crates.io-index)" = "cf3d2011ab5c909338f7887f4fc896d35932e29146c12c8d01da6b22a80ba759" -"checksum proc-macro2 1.0.18 (registry+https://github.com/rust-lang/crates.io-index)" = "beae6331a816b1f65d04c45b078fd8e6c93e8071771f41b8163255bbd8d7c8fa" +"checksum proc-macro2 1.0.19 (registry+https://github.com/rust-lang/crates.io-index)" = "04f5f085b5d71e2188cb8271e5da0161ad52c3f227a661a3c135fdf28e258b12" "checksum quote 0.6.13 (registry+https://github.com/rust-lang/crates.io-index)" = "6ce23b6b870e8f94f81fb0a363d65d86675884b34a09043c81e5562f11c1f8e1" "checksum quote 1.0.7 (registry+https://github.com/rust-lang/crates.io-index)" = "aa563d17ecb180e500da1cfd2b028310ac758de548efdd203e18f283af693f37" "checksum rand 0.6.5 (registry+https://github.com/rust-lang/crates.io-index)" = "6d71dacdc3c88c1fde3885a3be3fbab9f35724e6ce99467f7d9c5026132184ca" @@ -1543,7 +1553,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" "checksum rand_xorshift 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "77d416b86801d23dde1aa643023b775c3a462efc0ed96443add11546cdf1dca8" "checksum raw-window-handle 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)" = "0a441a7a6c80ad6473bd4b74ec1c9a4c951794285bf941c2126f607c72e48211" "checksum rdrand 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "678054eb77286b51581ba43620cc911abf02758c91f93f479767aed0f90458b2" -"checksum redox_syscall 0.1.56 (registry+https://github.com/rust-lang/crates.io-index)" = "2439c63f3f6139d1b57529d16bc3b8bb855230c8efcc5d3a896c8bea7c3b1e84" +"checksum redox_syscall 0.1.57 (registry+https://github.com/rust-lang/crates.io-index)" = "41cc0f7e4d5d4544e8861606a285bb08d3e70712ccc7d2b84d7c0ccfaf4b05ce" "checksum regex 1.3.9 (registry+https://github.com/rust-lang/crates.io-index)" = "9c3780fcf44b193bc4d09f36d2a3c87b251da4a046c87795a0d35f4f927ad8e6" "checksum regex-syntax 0.6.18 (registry+https://github.com/rust-lang/crates.io-index)" = "26412eb97c6b088a6997e05f69403a802a92d520de2f8e63c2b65f9e0f47c4e8" "checksum rustc-demangle 0.1.16 (registry+https://github.com/rust-lang/crates.io-index)" = "4c691c0e608126e00913e33f0ccf3727d5fc84573623b8d65b2df340b5201783" @@ -1552,20 +1562,20 @@ source = "registry+https://github.com/rust-lang/crates.io-index" "checksum same-file 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)" = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" "checksum scoped-tls 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ea6a9290e3c9cf0f18145ef7ffa62d68ee0bf5fcd651017e586dc7fd5da448c2" "checksum scopeguard 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd" -"checksum serde 1.0.112 (registry+https://github.com/rust-lang/crates.io-index)" = "736aac72d1eafe8e5962d1d1c3d99b0df526015ba40915cb3c49d042e92ec243" -"checksum serde_derive 1.0.112 (registry+https://github.com/rust-lang/crates.io-index)" = "bf0343ce212ac0d3d6afd9391ac8e9c9efe06b533c8d33f660f6390cc4093f57" +"checksum serde 1.0.115 (registry+https://github.com/rust-lang/crates.io-index)" = "e54c9a88f2da7238af84b5101443f0c0d0a3bbdc455e34a5c9497b1903ed55d5" +"checksum serde_derive 1.0.115 (registry+https://github.com/rust-lang/crates.io-index)" = "609feed1d0a73cc36a0182a840a9b37b4a82f0b1150369f0536a9e3f2a31dc48" "checksum shared_library 0.1.9 (registry+https://github.com/rust-lang/crates.io-index)" = "5a9e7e0f2bfae24d8a5b5a66c5b257a83c7412304311512a0c054cd5e619da11" "checksum slab 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)" = "c111b5bd5695e56cffe5129854aa230b39c93a305372fdbb2668ca2394eea9f8" -"checksum smallvec 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "c7cb5678e1615754284ec264d9bb5b4c27d2018577fd90ac0ceb578591ed5ee4" +"checksum smallvec 1.4.2 (registry+https://github.com/rust-lang/crates.io-index)" = "fbee7696b84bbf3d89a1c2eccff0850e3047ed46bfcd2e92c29a2d074d57e252" "checksum smithay-client-toolkit 0.6.6 (registry+https://github.com/rust-lang/crates.io-index)" = "421c8dc7acf5cb205b88160f8b4cc2c5cfabe210e43b2f80f009f4c1ef910f1d" "checksum stb_truetype 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)" = "f77b6b07e862c66a9f3e62a07588fee67cd90a9135a2b942409f195507b4fb51" "checksum strsim 0.9.3 (registry+https://github.com/rust-lang/crates.io-index)" = "6446ced80d6c486436db5c078dde11a9f73d42b57fb273121e160b84f63d894c" -"checksum syn 1.0.31 (registry+https://github.com/rust-lang/crates.io-index)" = "b5304cfdf27365b7585c25d4af91b35016ed21ef88f17ced89c7093b43dba8b6" +"checksum syn 1.0.38 (registry+https://github.com/rust-lang/crates.io-index)" = "e69abc24912995b3038597a7a593be5053eb0fb44f3cc5beec0deb421790c1f4" "checksum takeable-option 0.5.0 (registry+https://github.com/rust-lang/crates.io-index)" = "36ae8932fcfea38b7d3883ae2ab357b0d57a02caaa18ebb4f5ece08beaec4aa0" "checksum thread_local 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)" = "d40c6d1b69745a6ec6fb1ca717914848da4b44ae29d9b3080cbee91d72a69b14" "checksum toml 0.5.6 (registry+https://github.com/rust-lang/crates.io-index)" = "ffc92d160b1eef40665be3a05630d003936a3bc7da7421277846c2613e92c71a" "checksum unicode-xid 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "fc72304796d0818e357ead4e000d19c9c174ab23dc11093ac919054d20a6a7fc" -"checksum unicode-xid 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "826e7639553986605ec5979c7dd957c7895e93eabed50ab2ffa7f6128a75097c" +"checksum unicode-xid 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "f7fe0bb3479651439c9112f72b6c505038574c9fbb575ed1bf3b797fa39dd564" "checksum void 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)" = "6a02e4885ed3bc0f2de90ea6dd45ebcbb66dacffe03547fadbb0eeae2770887d" "checksum walkdir 2.3.1 (registry+https://github.com/rust-lang/crates.io-index)" = "777182bc735b6424e1a57516d35ed72cb8019d85c8c9bf536dccb3445c1a2f7d" "checksum wasi 0.9.0+wasi-snapshot-preview1 (registry+https://github.com/rust-lang/crates.io-index)" = "cccddf32554fecc6acb585f82a32a72e28b48f8c4c1883ddfeeeaa96f7d8e519" @@ -1575,7 +1585,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" "checksum wayland-scanner 0.23.6 (registry+https://github.com/rust-lang/crates.io-index)" = "93b02247366f395b9258054f964fe293ddd019c3237afba9be2ccbe9e1651c3d" "checksum wayland-sys 0.23.6 (registry+https://github.com/rust-lang/crates.io-index)" = "d94e89a86e6d6d7c7c9b19ebf48a03afaac4af6bc22ae570e9a24124b75358f4" "checksum winapi 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)" = "167dc9d6949a9b857f3451275e911c3f44255842c1f7a76f33c55103a909087a" -"checksum winapi 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)" = "8093091eeb260906a183e6ae1abdba2ef5ef2257a21801128899c3fc699229c6" +"checksum winapi 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)" = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" "checksum winapi-build 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "2d315eee3b34aca4797b2da6b13ed88266e6d612562a0c46390af8299fc699bc" "checksum winapi-i686-pc-windows-gnu 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" "checksum winapi-util 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)" = "70ec6ce85bb158151cae5e5c87f95a8e97d2c0c4b001223f33a334e3ce5de178"