diff --git a/esp-hal/CHANGELOG.md b/esp-hal/CHANGELOG.md index 4315cb68fbe..375bdb28fee 100644 --- a/esp-hal/CHANGELOG.md +++ b/esp-hal/CHANGELOG.md @@ -30,6 +30,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Improved interrupt latency on RISC-V based chips (#1679) - `esp_wifi::initialize` no longer requires running maximum CPU clock, instead check it runs above 80MHz. (#1688) - Move DMA descriptors from DMA Channel to each individual peripheral driver. (#1719) +- Improved interrupt latency on Xtensa based chips (#1735) ### Removed - uart: Removed `configure_pins` methods (#1592) diff --git a/esp-hal/ld/esp32/esp32.x b/esp-hal/ld/esp32/esp32.x index cd8d11c4411..e3c639b6112 100644 --- a/esp-hal/ld/esp32/esp32.x +++ b/esp-hal/ld/esp32/esp32.x @@ -8,6 +8,10 @@ PROVIDE(__zero_bss = default_mem_hook); PROVIDE(__init_data = default_mem_hook); PROVIDE(__post_init = default_post_init); +PROVIDE(__level_1_interrupt = handle_interrupts); +PROVIDE(__level_2_interrupt = handle_interrupts); +PROVIDE(__level_3_interrupt = handle_interrupts); + INCLUDE exception.x /* ESP32 fixups */ diff --git a/esp-hal/ld/esp32s2/esp32s2.x b/esp-hal/ld/esp32s2/esp32s2.x index 30e032c3e38..dc2e704197c 100644 --- a/esp-hal/ld/esp32s2/esp32s2.x +++ b/esp-hal/ld/esp32s2/esp32s2.x @@ -8,6 +8,10 @@ PROVIDE(__zero_bss = default_mem_hook); PROVIDE(__init_data = default_mem_hook); PROVIDE(__post_init = default_post_init); +PROVIDE(__level_1_interrupt = handle_interrupts); +PROVIDE(__level_2_interrupt = handle_interrupts); +PROVIDE(__level_3_interrupt = handle_interrupts); + INCLUDE exception.x /* Fixups for esp32s2 */ diff --git a/esp-hal/ld/esp32s3/esp32s3.x b/esp-hal/ld/esp32s3/esp32s3.x index 731d279cc3e..085b23b134d 100644 --- a/esp-hal/ld/esp32s3/esp32s3.x +++ b/esp-hal/ld/esp32s3/esp32s3.x @@ -7,6 +7,10 @@ PROVIDE(__zero_bss = default_mem_hook); PROVIDE(__init_data = default_mem_hook); PROVIDE(__post_init = default_post_init); +PROVIDE(__level_1_interrupt = handle_interrupts); +PROVIDE(__level_2_interrupt = handle_interrupts); +PROVIDE(__level_3_interrupt = handle_interrupts); + INCLUDE exception.x /* ESP32S3 fixups */ diff --git a/esp-hal/src/interrupt/mod.rs b/esp-hal/src/interrupt/mod.rs index bb74e9b06a2..8f21c4dfd9d 100644 --- a/esp-hal/src/interrupt/mod.rs +++ b/esp-hal/src/interrupt/mod.rs @@ -68,6 +68,8 @@ //! [`RESERVED_INTERRUPTS`]. #![warn(missing_docs)] +use core::ops::BitAnd; + #[cfg(riscv)] pub use self::riscv::*; #[cfg(xtensa)] @@ -114,3 +116,123 @@ impl InterruptHandler { (self.f)() } } + +#[cfg(large_intr_status)] +const STATUS_WORDS: usize = 3; + +#[cfg(very_large_intr_status)] +const STATUS_WORDS: usize = 4; + +#[cfg(not(any(large_intr_status, very_large_intr_status)))] +const STATUS_WORDS: usize = 2; + +/// Representation of peripheral-interrupt status bits. +#[derive(Clone, Copy, Default, Debug)] +pub struct InterruptStatus { + status: [u32; STATUS_WORDS], +} + +impl InterruptStatus { + const fn empty() -> Self { + InterruptStatus { + status: [0u32; STATUS_WORDS], + } + } + + #[cfg(large_intr_status)] + const fn from(w0: u32, w1: u32, w2: u32) -> Self { + Self { + status: [w0, w1, w2], + } + } + + #[cfg(very_large_intr_status)] + const fn from(w0: u32, w1: u32, w2: u32, w3: u32) -> Self { + Self { + status: [w0, w1, w2, w3], + } + } + + #[cfg(not(any(large_intr_status, very_large_intr_status)))] + const fn from(w0: u32, w1: u32) -> Self { + Self { status: [w0, w1] } + } + + /// Is the given interrupt bit set + pub fn is_set(&self, interrupt: u16) -> bool { + (self.status[interrupt as usize / 32] & (1 << (interrupt as u32 % 32))) != 0 + } + + /// Set the given interrupt status bit + pub fn set(&mut self, interrupt: u16) { + self.status[interrupt as usize / 32] |= 1 << (interrupt as u32 % 32); + } + + /// Return an iterator over the set interrupt status bits + pub fn iterator(&self) -> InterruptStatusIterator { + InterruptStatusIterator { + status: *self, + idx: 0, + } + } +} + +impl BitAnd for InterruptStatus { + type Output = InterruptStatus; + + fn bitand(self, rhs: Self) -> Self::Output { + #[cfg(large_intr_status)] + return Self::Output { + status: [ + self.status[0] & rhs.status[0], + self.status[1] & rhs.status[1], + self.status[2] & rhs.status[2], + ], + }; + + #[cfg(very_large_intr_status)] + return Self::Output { + status: [ + self.status[0] & rhs.status[0], + self.status[1] & rhs.status[1], + self.status[2] & rhs.status[2], + self.status[3] & rhs.status[3], + ], + }; + + #[cfg(not(any(large_intr_status, very_large_intr_status)))] + return Self::Output { + status: [ + self.status[0] & rhs.status[0], + self.status[1] & rhs.status[1], + ], + }; + } +} + +/// Iterator over set interrupt status bits +pub struct InterruptStatusIterator { + status: InterruptStatus, + idx: usize, +} + +impl Iterator for InterruptStatusIterator { + type Item = u8; + + fn next(&mut self) -> Option { + if self.idx == usize::MAX { + return None; + } + + for i in self.idx..STATUS_WORDS { + if self.status.status[i] != 0 { + let bit = self.status.status[i].trailing_zeros(); + self.idx = i; + self.status.status[i] &= !1 << bit; + return Some((bit + 32 * i as u32) as u8); + } + } + self.idx = usize::MAX; + None + } +} diff --git a/esp-hal/src/interrupt/riscv.rs b/esp-hal/src/interrupt/riscv.rs index 066fe3f075d..afc5a4c92e3 100644 --- a/esp-hal/src/interrupt/riscv.rs +++ b/esp-hal/src/interrupt/riscv.rs @@ -12,8 +12,6 @@ //! interrupt15() => Priority::Priority15 //! ``` -use core::ops::BitAnd; - pub use esp_riscv_rt::TrapFrame; use riscv::register::{mcause, mtvec}; @@ -24,6 +22,7 @@ pub use self::clic::*; #[cfg(plic)] pub use self::plic::*; pub use self::vectored::*; +use super::InterruptStatus; use crate::{ peripherals::{self, Interrupt}, Cpu, @@ -231,131 +230,6 @@ pub fn disable(_core: Cpu, interrupt: Interrupt) { } } -#[cfg(large_intr_status)] -const STATUS_WORDS: usize = 3; - -#[cfg(very_large_intr_status)] -const STATUS_WORDS: usize = 4; - -#[cfg(not(any(large_intr_status, very_large_intr_status)))] -const STATUS_WORDS: usize = 2; - -/// Representation of peripheral-interrupt status bits. -#[derive(Clone, Copy, Default, Debug)] -pub struct InterruptStatus { - status: [u32; STATUS_WORDS], -} - -impl InterruptStatus { - const fn empty() -> Self { - InterruptStatus { - #[cfg(large_intr_status)] - status: [0, 0, 0], - #[cfg(very_large_intr_status)] - status: [0, 0, 0, 0], - #[cfg(not(any(large_intr_status, very_large_intr_status)))] - status: [0, 0], - } - } - - #[cfg(large_intr_status)] - fn from(w0: u32, w1: u32, w2: u32) -> Self { - Self { - status: [w0, w1, w2], - } - } - - #[cfg(very_large_intr_status)] - fn from(w0: u32, w1: u32, w2: u32, w3: u32) -> Self { - Self { - status: [w0, w1, w2, w3], - } - } - - #[cfg(not(any(large_intr_status, very_large_intr_status)))] - fn from(w0: u32, w1: u32) -> Self { - Self { status: [w0, w1] } - } - - /// Is the given interrupt bit set - pub fn is_set(&self, interrupt: u16) -> bool { - (self.status[interrupt as usize / 32] & (1 << (interrupt as u32 % 32))) != 0 - } - - /// Set the given interrupt status bit - pub fn set(&mut self, interrupt: u16) { - self.status[interrupt as usize / 32] |= 1 << (interrupt as u32 % 32); - } - - /// Return an iterator over the set interrupt status bits - pub fn iterator(&self) -> InterruptStatusIterator { - InterruptStatusIterator { - status: *self, - idx: 0, - } - } -} - -impl BitAnd for InterruptStatus { - type Output = InterruptStatus; - - fn bitand(self, rhs: Self) -> Self::Output { - #[cfg(large_intr_status)] - return Self::Output { - status: [ - self.status[0] & rhs.status[0], - self.status[1] & rhs.status[1], - self.status[2] & rhs.status[2], - ], - }; - - #[cfg(very_large_intr_status)] - return Self::Output { - status: [ - self.status[0] & rhs.status[0], - self.status[1] & rhs.status[1], - self.status[2] & rhs.status[2], - self.status[3] & rhs.status[3], - ], - }; - - #[cfg(not(any(large_intr_status, very_large_intr_status)))] - return Self::Output { - status: [ - self.status[0] & rhs.status[0], - self.status[1] & rhs.status[1], - ], - }; - } -} - -/// Iterator over set interrupt status bits -pub struct InterruptStatusIterator { - status: InterruptStatus, - idx: usize, -} - -impl Iterator for InterruptStatusIterator { - type Item = u8; - - fn next(&mut self) -> Option { - if self.idx == usize::MAX { - return None; - } - - for i in self.idx..STATUS_WORDS { - if self.status.status[i] != 0 { - let bit = self.status.status[i].trailing_zeros(); - self.idx = i; - self.status.status[i] &= !1 << bit; - return Some((bit + 32 * i as u32) as u8); - } - } - self.idx = usize::MAX; - None - } -} - /// Get status of peripheral interrupts #[inline] pub fn get_status(_core: Cpu) -> InterruptStatus { @@ -538,7 +412,7 @@ mod vectored { // this has no effect on level interrupts, but the interrupt may be an edge one // so we clear it anyway - clear(crate::get_core(), cpu_intr); + clear(core, cpu_intr); let configured_interrupts = get_configured_interrupts(core, status, unsafe { core::mem::transmute(INTERRUPT_TO_PRIORITY[cpu_intr as usize - 1] as u8) diff --git a/esp-hal/src/interrupt/xtensa.rs b/esp-hal/src/interrupt/xtensa.rs index 039cd434b2b..328d225ca85 100644 --- a/esp-hal/src/interrupt/xtensa.rs +++ b/esp-hal/src/interrupt/xtensa.rs @@ -2,6 +2,7 @@ use xtensa_lx::interrupt::{self, InterruptNumber}; use xtensa_lx_rt::exception::Context; pub use self::vectored::*; +use super::InterruptStatus; use crate::{ peripherals::{self, Interrupt}, Cpu, @@ -149,66 +150,86 @@ pub fn clear(_core: Cpu, which: CpuInterrupt) { } /// Get status of peripheral interrupts -pub fn get_status(core: Cpu) -> u128 { +#[cfg(large_intr_status)] +pub fn get_status(core: Cpu) -> InterruptStatus { unsafe { - let status = match core { - Cpu::ProCpu => { - ((*core0_interrupt_peripheral()) + match core { + Cpu::ProCpu => InterruptStatus::from( + (*core0_interrupt_peripheral()) .pro_intr_status_0() .read() - .bits() as u128) - | ((*core0_interrupt_peripheral()) - .pro_intr_status_1() - .read() - .bits() as u128) - << 32 - | ((*core0_interrupt_peripheral()) - .pro_intr_status_2() - .read() - .bits() as u128) - << 64 - } + .bits(), + (*core0_interrupt_peripheral()) + .pro_intr_status_1() + .read() + .bits(), + (*core0_interrupt_peripheral()) + .pro_intr_status_2() + .read() + .bits(), + ), #[cfg(multi_core)] - Cpu::AppCpu => { - ((*core1_interrupt_peripheral()) + Cpu::AppCpu => InterruptStatus::from( + (*core1_interrupt_peripheral()) .app_intr_status_0() .read() - .bits() as u128) - | ((*core1_interrupt_peripheral()) - .app_intr_status_1() - .read() - .bits() as u128) - << 32 - | ((*core1_interrupt_peripheral()) - .app_intr_status_2() - .read() - .bits() as u128) - << 64 - } - }; + .bits(), + (*core1_interrupt_peripheral()) + .app_intr_status_1() + .read() + .bits(), + (*core1_interrupt_peripheral()) + .app_intr_status_2() + .read() + .bits(), + ), + } + } +} - #[cfg(esp32s3)] - let status = match core { - Cpu::ProCpu => { - status - | ((*core0_interrupt_peripheral()) - .pro_intr_status_3() - .read() - .bits() as u128) - << 96 - } +/// Get status of peripheral interrupts +#[cfg(very_large_intr_status)] +pub fn get_status(core: Cpu) -> InterruptStatus { + unsafe { + match core { + Cpu::ProCpu => InterruptStatus::from( + (*core0_interrupt_peripheral()) + .pro_intr_status_0() + .read() + .bits(), + (*core0_interrupt_peripheral()) + .pro_intr_status_1() + .read() + .bits(), + (*core0_interrupt_peripheral()) + .pro_intr_status_2() + .read() + .bits(), + (*core0_interrupt_peripheral()) + .pro_intr_status_3() + .read() + .bits(), + ), #[cfg(multi_core)] - Cpu::AppCpu => { - status - | ((*core1_interrupt_peripheral()) - .app_intr_status_3() - .read() - .bits() as u128) - << 96 - } - }; - - status + Cpu::AppCpu => InterruptStatus::from( + (*core1_interrupt_peripheral()) + .app_intr_status_0() + .read() + .bits(), + (*core1_interrupt_peripheral()) + .app_intr_status_1() + .read() + .bits(), + (*core1_interrupt_peripheral()) + .app_intr_status_2() + .read() + .bits(), + (*core1_interrupt_peripheral()) + .app_intr_status_3() + .read() + .bits(), + ), + } } } @@ -310,8 +331,12 @@ mod vectored { } /// Get the interrupts configured for the core - #[inline] - fn get_configured_interrupts(core: Cpu, mut status: u128) -> [u128; 8] { + #[inline(always)] + fn get_configured_interrupts( + core: Cpu, + status: InterruptStatus, + level: u32, + ) -> InterruptStatus { unsafe { let intr_map_base = match core { Cpu::ProCpu => (*core0_interrupt_peripheral()).pro_mac_intr_map().as_ptr(), @@ -319,22 +344,22 @@ mod vectored { Cpu::AppCpu => (*core1_interrupt_peripheral()).app_mac_intr_map().as_ptr(), }; - let mut levels = [0u128; 8]; + let mut res = InterruptStatus::empty(); - while status != 0 { - let interrupt_nr = status.trailing_zeros(); + for interrupt_nr in status.iterator() { let i = interrupt_nr as isize; let cpu_interrupt = intr_map_base.offset(i).read_volatile(); // safety: cast is safe because of repr(u32) let cpu_interrupt: CpuInterrupt = core::mem::transmute::(cpu_interrupt); - let level = cpu_interrupt.level() as u8 as usize; + let int_level = cpu_interrupt.level() as u8 as u32; - levels[level] |= 1 << i; - status &= !(1u128 << interrupt_nr); + if int_level == level { + res.set(interrupt_nr as u16); + } } - levels + res } } @@ -417,26 +442,14 @@ mod vectored { }) } - #[no_mangle] - #[link_section = ".rwtext"] - unsafe fn __level_1_interrupt(level: u32, save_frame: &mut Context) { - handle_interrupts(level, save_frame) - } + // The linker script defines `__level_1_interrupt`, `__level_2_interrupt` and + // `__level_3_interrupt` as `handle_interrupts` #[no_mangle] - #[link_section = ".rwtext"] - unsafe fn __level_2_interrupt(level: u32, save_frame: &mut Context) { - handle_interrupts(level, save_frame) - } - - #[no_mangle] - #[link_section = ".rwtext"] - unsafe fn __level_3_interrupt(level: u32, save_frame: &mut Context) { - handle_interrupts(level, save_frame) - } - #[ram] unsafe fn handle_interrupts(level: u32, save_frame: &mut Context) { + let core = crate::get_core(); + let cpu_interrupt_mask = interrupt::get() & interrupt::get_mask() & CPU_INTERRUPT_LEVELS[level as usize]; @@ -458,30 +471,23 @@ mod vectored { // for edge interrupts cannot rely on the interrupt status // register, therefore call all registered // handlers for current level - let interrupt_levels = - get_configured_interrupts(crate::get_core(), chip_specific::INTERRUPT_EDGE); - let interrupt_mask = interrupt_levels[level as usize]; - let mut interrupt_mask = interrupt_mask & chip_specific::INTERRUPT_EDGE; - loop { - let interrupt_nr = interrupt_mask.trailing_zeros(); - if let Ok(interrupt) = peripherals::Interrupt::try_from(interrupt_nr as u16) { - handle_interrupt(level, interrupt, save_frame) - } else { - break; - } - interrupt_mask &= !(1u128 << interrupt_nr); + let configured_interrupts = + get_configured_interrupts(core, chip_specific::INTERRUPT_EDGE, level); + + for interrupt_nr in configured_interrupts.iterator() { + // Don't use `Interrupt::try_from`. It's slower and placed in flash + let interrupt = unsafe { core::mem::transmute(interrupt_nr as u16) }; + handle_interrupt(level, interrupt, save_frame); } } else { - // finally check periperal sources and fire of handlers from pac + // finally check peripheral sources and fire of handlers from pac // peripheral mapped interrupts are cleared by the peripheral - let status = get_status(crate::get_core()); - let interrupt_levels = get_configured_interrupts(crate::get_core(), status); - let interrupt_mask = status & interrupt_levels[level as usize]; - let interrupt_nr = interrupt_mask.trailing_zeros(); - - // Interrupt::try_from can fail if interrupt already de-asserted: - // silently ignore - if let Ok(interrupt) = peripherals::Interrupt::try_from(interrupt_nr as u16) { + let status = get_status(core); + let configured_interrupts = get_configured_interrupts(core, status, level); + + for interrupt_nr in configured_interrupts.iterator() { + // Don't use `Interrupt::try_from`. It's slower and placed in flash + let interrupt = unsafe { core::mem::transmute(interrupt_nr as u16) }; handle_interrupt(level, interrupt, save_frame); } } @@ -511,8 +517,11 @@ mod vectored { #[cfg(esp32)] mod chip_specific { use super::*; - pub const INTERRUPT_EDGE: u128 = - 0b_0000_0000_0000_0000_0000_0000_0000_0000__0000_0000_0000_0000_0000_0000_0000_0011_1111_1100_0000_0000_0000_0000_0000_0000__0000_0000_0000_0000_0000_0000_0000_0000; + pub const INTERRUPT_EDGE: InterruptStatus = InterruptStatus::from( + 0b0000_0000_0000_0000_0000_0000_0000_0011, + 0b1111_1100_0000_0000_0000_0000_0000_0000, + 0b0000_0000_0000_0000_0000_0000_0000_0000, + ); #[inline] pub fn interrupt_is_edge(interrupt: Interrupt) -> bool { use peripherals::Interrupt::*; @@ -534,8 +543,11 @@ mod vectored { #[cfg(esp32s2)] mod chip_specific { use super::*; - pub const INTERRUPT_EDGE: u128 = - 0b_0000_0000_0000_0000_0000_0000_0000_0000__0000_0000_0000_0000_0000_0011_1011_1111_1100_0000_0000_0000_0000_0000_0000_0000__0000_0000_0000_0000_0000_0000_0000_0000; + pub const INTERRUPT_EDGE: InterruptStatus = InterruptStatus::from( + 0b0000_0000_0000_0000_0000_0011_1011_1111, + 0b1100_0000_0000_0000_0000_0000_0000_0000, + 0b0000_0000_0000_0000_0000_0000_0000_0000, + ); #[inline] pub fn interrupt_is_edge(interrupt: Interrupt) -> bool { use peripherals::Interrupt::*; @@ -559,7 +571,7 @@ mod vectored { #[cfg(esp32s3)] mod chip_specific { use super::*; - pub const INTERRUPT_EDGE: u128 = 0; + pub const INTERRUPT_EDGE: InterruptStatus = InterruptStatus::empty(); #[inline] pub fn interrupt_is_edge(_interrupt: Interrupt) -> bool { false diff --git a/esp-metadata/devices/esp32.toml b/esp-metadata/devices/esp32.toml index 8e02cb7cbc8..53d81d9f8e9 100644 --- a/esp-metadata/devices/esp32.toml +++ b/esp-metadata/devices/esp32.toml @@ -60,6 +60,7 @@ symbols = [ "wifi", "psram", "timg_timer1", + "large_intr_status", # ROM capabilities "rom_crc_le", diff --git a/esp-metadata/devices/esp32s2.toml b/esp-metadata/devices/esp32s2.toml index 2d11f3e9203..9d51c04ec04 100644 --- a/esp-metadata/devices/esp32s2.toml +++ b/esp-metadata/devices/esp32s2.toml @@ -58,6 +58,7 @@ symbols = [ "psram", "ulp_riscv_core", "timg_timer1", + "large_intr_status", # ROM capabilities "rom_crc_le", diff --git a/esp-metadata/devices/esp32s3.toml b/esp-metadata/devices/esp32s3.toml index 0377f118e7f..7b592e239e5 100644 --- a/esp-metadata/devices/esp32s3.toml +++ b/esp-metadata/devices/esp32s3.toml @@ -70,6 +70,7 @@ symbols = [ "psram", "ulp_riscv_core", "timg_timer1", + "very_large_intr_status", # ROM capabilities "rom_crc_le",