diff --git a/arch/x86/apic.c b/arch/x86/apic.c index 9ff8c97b..8bd9e8f9 100644 --- a/arch/x86/apic.c +++ b/arch/x86/apic.c @@ -25,7 +25,6 @@ #include #include #include -#include #include #include #include @@ -179,6 +178,9 @@ void init_apic_timer(void) { apic_lvt_timer_t timer; printk("Initializing local APIC timer\n"); + /* Enable interrupts for calibration */ + unsigned long flags = interrupts_enable_save(); + /* Spend 200ms calibrating the timer, 10 iterations of 20ms each */ for (i = 0; i < CAL_ITERATIONS; ++i) { /* Set the counter to the max value (0xFFFFFFFF) */ @@ -197,6 +199,8 @@ void init_apic_timer(void) { min_ticks = min(min_ticks, elapsed_ticks); } + interrupts_restore(flags); + /* Interrupt every min_ticks ticks */ apic_write(APIC_TMR_DCR, APIC_TIMER_DIVIDE_BY_16); apic_write(APIC_TMR_ICR, min_ticks); @@ -206,5 +210,5 @@ void init_apic_timer(void) { timer.timer_mode = APIC_LVT_TIMER_PERIODIC; apic_write(APIC_LVT_TIMER, timer.reg); - pit_disable(); + PERCPU_SET(apic_timer_enabled, true); } diff --git a/arch/x86/entry.S b/arch/x86/entry.S index 48ac24a5..f160af04 100644 --- a/arch/x86/entry.S +++ b/arch/x86/entry.S @@ -162,6 +162,7 @@ GLOBAL(end_exception_handlers) GLOBAL(interrupt_handlers) interrupt_handler timer timer_interrupt_handler timer_irq +interrupt_handler apic_timer apic_timer_interrupt_handler apic_timer_irq interrupt_handler uart1 uart_interrupt_handler serial_com1_irq interrupt_handler uart2 uart_interrupt_handler serial_com2_irq interrupt_handler keyboard keyboard_interrupt_handler kb_port1_irq diff --git a/arch/x86/traps.c b/arch/x86/traps.c index 7750d6f7..8b689ffc 100644 --- a/arch/x86/traps.c +++ b/arch/x86/traps.c @@ -44,6 +44,7 @@ extern void asm_interrupt_handler_uart2(void); extern void asm_interrupt_handler_keyboard(void); extern void asm_interrupt_handler_timer(void); extern void asm_interrupt_handler_dummy(void); +extern void asm_interrupt_handler_apic_timer(void); extern void terminate_user_task(void); @@ -165,7 +166,7 @@ void init_traps(const cpu_t *cpu) { set_intr_gate(&percpu->idt[KB_PORT1_IRQ], __KERN_CS, _ul(asm_interrupt_handler_keyboard), GATE_DPL0, GATE_PRESENT, 0); set_intr_gate(&percpu->idt[APIC_TIMER_IRQ], __KERN_CS, - _ul(asm_interrupt_handler_timer), GATE_DPL0, GATE_PRESENT, 0); + _ul(asm_interrupt_handler_apic_timer), GATE_DPL0, GATE_PRESENT, 0); set_intr_gate(&percpu->idt[APIC_SPI_VECTOR], __KERN_CS, _ul(asm_interrupt_handler_dummy), GATE_DPL0, GATE_PRESENT, 0); diff --git a/common/acpi.c b/common/acpi.c index f036fcd1..68f8cc38 100644 --- a/common/acpi.c +++ b/common/acpi.c @@ -750,7 +750,7 @@ ACPI_STATUS init_acpi(void) { void acpi_power_off(void) { AcpiEnterSleepStatePrep(ACPI_STATE_S5); - cli(); + interrupts_disable(); AcpiEnterSleepState(ACPI_STATE_S5); panic("Power Off"); } diff --git a/common/setup.c b/common/setup.c index 0976a787..39e9dbb7 100644 --- a/common/setup.c +++ b/common/setup.c @@ -165,18 +165,18 @@ static void __text_init init_vga_console(void) { } void __text_init init_timers(cpu_t *cpu) { - static bool __data_init hpet_initialized = false; - if (is_cpu_bsp(cpu)) { if (opt_hpet) - hpet_initialized = init_hpet(cpu); + boot_flags.timer_global = init_hpet(cpu); - if (!hpet_initialized && opt_pit) + if (!boot_flags.timer_global && opt_pit) { init_pit(cpu); + boot_flags.timer_global = true; + } } if (opt_apic_timer) { - if (hpet_initialized || opt_pit) /* Needed for APIC timer calibration */ + if (boot_flags.timer_global) /* Needed for APIC timer calibration */ init_apic_timer(); else { warning("CPU%u: Unable to initialize APIC timer - no calibration timers!", @@ -214,9 +214,6 @@ void __noreturn __text_init kernel_start(uint32_t multiboot_magic, unsigned long /* Initialize Programmable Interrupt Controller */ init_pic(); - /* PIC is initialized - enable local interrupts */ - sti(); - /* Initialize Physical Memory Manager */ init_regions(); init_pmm(); @@ -264,8 +261,9 @@ void __noreturn __text_init kernel_start(uint32_t multiboot_magic, unsigned long init_ioapic(); - /* Initialize timers */ + /* Initialize timers and enable interrupts */ init_timers(bsp); + interrupts_enable(); if (!boot_flags.nosmp) init_smp(); diff --git a/drivers/keyboard.c b/drivers/keyboard.c index f1bd9b46..1ff1f8c0 100644 --- a/drivers/keyboard.c +++ b/drivers/keyboard.c @@ -67,7 +67,7 @@ void keyboard_reboot(void) { if (!i8042_present) return; - cli(); + interrupts_disable(); while ((inb(KEYBOARD_PORT_CMD) & KEYBOARD_STATUS_IN_FULL) != 0) cpu_relax(); diff --git a/include/arch/x86/asm-macros.h b/include/arch/x86/asm-macros.h index 565bbdfa..908a3f7d 100644 --- a/include/arch/x86/asm-macros.h +++ b/include/arch/x86/asm-macros.h @@ -323,8 +323,10 @@ name ## _end: "pop %%" STR(_ASM_BX) "\n" #if defined(__x86_64__) +#define PUSHF() "pushfq\n" #define POPF() "popfq\n" #else +#define PUSHF() "pushf\n" #define POPF() "popf\n" #endif diff --git a/include/lib.h b/include/lib.h index d4bef3d8..0f5c47aa 100644 --- a/include/lib.h +++ b/include/lib.h @@ -171,22 +171,49 @@ static inline void int3(void) { asm volatile("int3"); } -static inline unsigned long read_flags(void) { +static inline unsigned long read_eflags(void) { unsigned long flags; - asm volatile( -#if defined(__i386__) - "pushfd;" - "popl %0" -#elif defined(__x86_64__) - "pushfq;" - "popq %0" -#endif - : "=r"(flags)); + asm volatile(PUSHF() "pop %0" : "=r"(flags)); + + return flags; +} + +static inline void write_eflags(unsigned long flags) { + asm volatile("push %0\n" POPF()::"r"(flags)); +} + +static inline void interrupts_enable(void) { + sti(); +} +static inline unsigned long interrupts_enable_save(void) { + unsigned long flags = read_eflags(); + interrupts_enable(); return flags; } +static inline void interrupts_disable(void) { + cli(); +} + +static inline unsigned long interrupts_disable_save(void) { + unsigned long flags = read_eflags(); + interrupts_disable(); + return flags; +} + +static inline void interrupts_restore(unsigned long flags) { + if (flags & X86_EFLAGS_IF) + sti(); + else + cli(); +} + +static inline bool interrupts_enabled(void) { + return read_eflags() & X86_EFLAGS_IF; +} + static inline unsigned long read_cs(void) { unsigned long cs; diff --git a/include/percpu.h b/include/percpu.h index b6891499..a61ea9e1 100644 --- a/include/percpu.h +++ b/include/percpu.h @@ -58,6 +58,8 @@ struct percpu { #endif unsigned long usermode_private; + volatile unsigned long apic_ticks; + bool apic_timer_enabled; } __aligned(PAGE_SIZE); typedef struct percpu percpu_t; diff --git a/include/setup.h b/include/setup.h index aeaf800a..6081983a 100644 --- a/include/setup.h +++ b/include/setup.h @@ -34,7 +34,7 @@ struct boot_flags { uint64_t virt : 1, legacy_devs : 1, i8042 : 1, vga : 1, msi : 1, aspm : 1, rtc : 1, - nosmp : 1, rsvd : 56; + nosmp : 1, timer_global : 1, rsvd : 55; }; typedef struct boot_flags boot_flags_t; diff --git a/include/time.h b/include/time.h index ef8c531b..726f7fb0 100644 --- a/include/time.h +++ b/include/time.h @@ -27,13 +27,19 @@ typedef uint64_t time_t; -extern void msleep(time_t ms); +extern int msleep(time_t ms); +extern int msleep_local(time_t ms); extern time_t get_timer_ticks(void); +extern time_t get_local_ticks(void); /* Static declarations */ -static inline void sleep(time_t s) { - msleep(s * 1000); +static inline int sleep(time_t s) { + return msleep(s * 1000); +} + +static inline int sleep_local(time_t s) { + return msleep_local(s * 1000); } #endif diff --git a/lib/lib.c b/lib/lib.c index 8ed0a77b..69d1c536 100644 --- a/lib/lib.c +++ b/lib/lib.c @@ -28,7 +28,7 @@ #include void __noreturn halt(void) { - cli(); + interrupts_disable(); while (1) { hlt(); @@ -40,7 +40,7 @@ void __noreturn halt(void) { void __noreturn hard_reboot(void) { idt_ptr_t idt_ptr = {0}; - cli(); + interrupts_disable(); lidt(&idt_ptr); int3(); diff --git a/lib/time.c b/lib/time.c index 26b0bcee..834ecb61 100644 --- a/lib/time.c +++ b/lib/time.c @@ -23,22 +23,56 @@ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #include +#include +#include +#include #include -static volatile time_t ticks = 0; +extern boot_flags_t boot_flags; + +static __aligned(16) volatile time_t ticks = 0; void timer_interrupt_handler(void) { - ++ticks; + asm volatile("lock incq %[ticks]" : [ ticks ] "=m"(ACCESS_ONCE(ticks))); + apic_EOI(); +} + +void apic_timer_interrupt_handler(void) { + asm volatile("lock incq %%gs:%[ticks]" + : [ ticks ] "=m"(ACCESS_ONCE(PERCPU_VAR(apic_ticks)))); apic_EOI(); } -void msleep(time_t ms) { - time_t end = ticks + ms; - while (ticks < end) { +int msleep(time_t ms) { + time_t end; + + if (!boot_flags.timer_global) + return -ENODEV; + + end = ACCESS_ONCE(ticks) + ms; + while (ACCESS_ONCE(ticks) < end) cpu_relax(); - } + + return 0; +} + +int msleep_local(time_t ms) { + time_t end; + + if (!PERCPU_GET(apic_timer_enabled)) + return -ENODEV; + + end = PERCPU_GET(apic_ticks) + ms; + while (PERCPU_GET(apic_ticks) < end) + cpu_relax(); + + return 0; } time_t get_timer_ticks(void) { - return ticks; + return ACCESS_ONCE(ticks); +} + +time_t get_local_ticks(void) { + return PERCPU_GET(apic_ticks); } diff --git a/smp/smp.c b/smp/smp.c index bff4b570..312d3915 100644 --- a/smp/smp.c +++ b/smp/smp.c @@ -56,8 +56,9 @@ void __noreturn ap_startup(void) { init_traps(cpu); init_apic(ap_cpuid, apic_get_mode()); + /* Initialize timers and enable interrupts */ init_timers(cpu); - sti(); + interrupts_enable(); if (opt_fpu) enable_fpu(); diff --git a/tests/test_cond_branch_mispredictions.c b/tests/test_cond_branch_mispredictions.c index 9f5f4d30..c9b9c15b 100644 --- a/tests/test_cond_branch_mispredictions.c +++ b/tests/test_cond_branch_mispredictions.c @@ -215,13 +215,13 @@ int __aligned(PAGE_SIZE) test_cond_branch_mispredictions(void *unused) { printk("Testing conditional branch %s BTB flushing\n", WITH_BTB_FLUSH ? "with" : "without"); - cli(); + unsigned long flags = interrupts_disable_save(); test_cond_forward_branch_cl0(LOOP_ITERATIONS); test_cond_forward_branch_cl1(LOOP_ITERATIONS); test_cond_backward_branch_cl0(LOOP_ITERATIONS); test_cond_backward_branch_cl1(LOOP_ITERATIONS); - sti(); + interrupts_restore(flags); return 0; } diff --git a/tests/test_uncond_branch_mispredictions.c b/tests/test_uncond_branch_mispredictions.c index d1720f85..6be92edf 100644 --- a/tests/test_uncond_branch_mispredictions.c +++ b/tests/test_uncond_branch_mispredictions.c @@ -231,13 +231,13 @@ int __aligned(PAGE_SIZE) test_uncond_branch_mispredictions(void *unused) { printk("Testing direct unconditional " STR(BRANCH) " %s BTB flushing\n", WITH_BTB_FLUSH ? "with" : "without"); - cli(); + unsigned long flags = interrupts_disable_save(); test_uncond_forward_branch_cl0(LOOP_ITERATIONS); test_uncond_forward_branch_cl1(LOOP_ITERATIONS); test_uncond_backward_branch_cl0(LOOP_ITERATIONS); test_uncond_backward_branch_cl1(LOOP_ITERATIONS); - sti(); + interrupts_restore(flags); return 0; }