Fix interrupts handling and improve syscall/exception stack handling

arch/x86/asm-offsets.c

@@ -23,8 +23,10 @@
  * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  */
 
+#include <acpi_ktf.h>
 #include <compiler.h>
 #include <percpu.h>
+#include <traps.h>
 
 #define EMIT_DEFINE(s, v)                                                                \
     asm volatile(".ascii \"@@#define " #s " %0 /* " #v " */@@\"\n" ::"i"(v))
@@ -34,14 +36,22 @@
 void __asm_offset_header(void) {
     OFFSETOF(usermode_private, percpu_t, usermode_private);
 
-    OFFSETOF(cpu_exc_error_code, cpu_exc_t, error_code);
     OFFSETOF(cpu_exc_vector, cpu_exc_t, vector);
+    OFFSETOF(cpu_exc_error_code, cpu_exc_t, error_code);
     OFFSETOF(cpu_exc_ip, cpu_exc_t, _ASM_IP);
     OFFSETOF(cpu_exc_cs, cpu_exc_t, cs);
     OFFSETOF(cpu_exc_flags, cpu_exc_t, _ASM_FLAGS);
     OFFSETOF(cpu_exc_sp, cpu_exc_t, _ASM_SP);
     OFFSETOF(cpu_exc_ss, cpu_exc_t, ss);
 
+    OFFSETOF(cpu_regs_ss, cpu_regs_t, exc.ss);
+    OFFSETOF(cpu_regs_sp, cpu_regs_t, exc._ASM_SP);
+    OFFSETOF(cpu_regs_flags, cpu_regs_t, exc._ASM_FLAGS);
+    OFFSETOF(cpu_regs_cs, cpu_regs_t, exc.cs);
+    OFFSETOF(cpu_regs_ip, cpu_regs_t, exc._ASM_IP);
+    OFFSETOF(cpu_regs_error_code, cpu_regs_t, exc.error_code);
+    OFFSETOF(cpu_regs_vector, cpu_regs_t, exc.vector);
+
     OFFSETOF(cpu_regs_ax, cpu_regs_t, _ASM_AX);
     OFFSETOF(cpu_regs_bx, cpu_regs_t, _ASM_BX);
     OFFSETOF(cpu_regs_cx, cpu_regs_t, _ASM_CX);
@@ -60,11 +70,11 @@ void __asm_offset_header(void) {
     OFFSETOF(cpu_regs_r15, cpu_regs_t, r15);
 #endif
 
-    OFFSETOF(cpu_regs_error_code, cpu_regs_t, exc.error_code);
-    OFFSETOF(cpu_regs_vector, cpu_regs_t, exc.vector);
-    OFFSETOF(cpu_regs_ip, cpu_regs_t, exc._ASM_IP);
-    OFFSETOF(cpu_regs_cs, cpu_regs_t, exc.cs);
-    OFFSETOF(cpu_regs_flags, cpu_regs_t, exc._ASM_FLAGS);
-    OFFSETOF(cpu_regs_sp, cpu_regs_t, exc._ASM_SP);
-    OFFSETOF(cpu_regs_ss, cpu_regs_t, exc.ss);
+    EMIT_DEFINE(serial_com1_irq, SERIAL_COM1_IRQ);
+    EMIT_DEFINE(serial_com2_irq, SERIAL_COM2_IRQ);
+    EMIT_DEFINE(timer_irq, TIMER_IRQ);
+    EMIT_DEFINE(kb_port1_irq, KB_PORT1_IRQ);
+    EMIT_DEFINE(kb_port2_irq, KB_PORT2_IRQ);
+    EMIT_DEFINE(apic_timer_irq, APIC_TIMER_IRQ);
+    EMIT_DEFINE(acpi_sci_irq, ACPI_SCI_IRQ);
 }

arch/x86/entry.S

@@ -32,70 +32,86 @@
 #include <usermode.h>
 #include <errno.h>
 
-.macro _handle_usermode cr3
+.macro MASK_USER_FLAGS
+    PUSHF
+    andl $~(USERMODE_FLAGS_MASK), (%_ASM_SP)
+    POPF
+.endm
+
+.macro _from_usermode switch_stack=0
+    SET_CR3 cr3
+    swapgs
+    .if \switch_stack == 1
+        SWITCH_STACK
+    .endif
+.endm
+
+.macro _to_usermode switch_stack=0
+    .if \switch_stack == 1
+        SWITCH_STACK
+    .endif
+    swapgs
+    SET_CR3 user_cr3
+.endm
+
+.macro cond_from_usermode
     testb $0x3, cpu_exc_cs(%_ASM_SP)
     jz 1f /* skip if from kernel mode */
-        SET_CR3 \cr3
-        swapgs
+        MASK_USER_FLAGS
+        _from_usermode
     1:
 .endm
 
-.macro enter_from_usermode
-    _handle_usermode cr3
+.macro cond_to_usermode
+    testb $0x3, cpu_exc_cs(%_ASM_SP)
+    jz 1f /* skip if from kernel mode */
+        _to_usermode
+    1:
 .endm
 
-.macro enter_to_usermode
-    _handle_usermode user_cr3
+.macro syscall_from_usermode switch_stack=1
+    _from_usermode switch_stack=\switch_stack
 .endm
 
-.macro syscall_from_usermode
-    SET_CR3 cr3
-    swapgs
-    SWITCH_STACK
-.endm
-
-.macro syscall_to_usermode
-    SWITCH_STACK
-    swapgs
-    SET_CR3 user_cr3
+.macro syscall_to_usermode switch_stack=1
+    _to_usermode switch_stack=\switch_stack
 .endm
 
 .macro exception_handler sym vec has_error_code
 ENTRY(entry_\sym)
-    enter_from_usermode
-
     .if \has_error_code == 0
         push $0
     .endif
-
-#if defined(__x86_64__)
-    movl $\vec, cpu_exc_vector(%_ASM_SP)
-#else
     push $\vec
-#endif
 
+    cond_from_usermode
     jmp handle_exception
 END_FUNC(entry_\sym)
 .endm
 
-.macro MASK_USER_FLAGS
-    PUSHF
-    andl $~(USERMODE_FLAGS_MASK), (%_ASM_SP)
-    POPF
+.macro check_syscall_exit
+    cmp $SYSCALL_EXIT, %_ASM_AX
+    jnz 1f
+        _from_usermode
+        jmp syscall_exit
+    1:
 .endm
 
-.macro interrupt_handler sym func
+.macro interrupt_handler sym func vec
 ENTRY(asm_interrupt_handler_\sym)
-    enter_from_usermode
-
-    MASK_USER_FLAGS
+    push $0 /* fake error_code */
+    push $\vec /* IRQ vector */
+    cond_from_usermode
 
     cld
     SAVE_ALL_REGS
     call \func
     RESTORE_ALL_REGS
 
-    enter_to_usermode
+    cond_to_usermode
+    /* Popping error_code and vector machine words added on entry. */
+    /* By using cpu_exc_t offset we make it arch-agnostic.         */
+    add $cpu_exc_ip, %_ASM_SP
     IRET
 END_FUNC(asm_interrupt_handler_\sym)
 .endm
@@ -109,9 +125,10 @@ ENTRY(handle_exception)
 
     RESTORE_ALL_REGS
 
-    add $8, %_ASM_SP
-
-    enter_to_usermode
+    cond_to_usermode
+    /* Popping error_code and vector machine words added on entry. */
+    /* By using cpu_exc_t offset we make it arch-agnostic.         */
+    add $cpu_exc_ip, %_ASM_SP
     IRET
 END_FUNC(handle_exception)
 
@@ -142,16 +159,18 @@ exception_handler SE  X86_EX_SE  1
 GLOBAL(end_exception_handlers)
 
 GLOBAL(interrupt_handlers)
-interrupt_handler timer timer_interrupt_handler
-interrupt_handler uart uart_interrupt_handler
-interrupt_handler keyboard keyboard_interrupt_handler
+interrupt_handler timer timer_interrupt_handler timer_irq
+interrupt_handler uart1 uart_interrupt_handler serial_com1_irq
+interrupt_handler uart2 uart_interrupt_handler serial_com2_irq
+interrupt_handler keyboard keyboard_interrupt_handler kb_port1_irq
 #ifdef KTF_ACPICA
-interrupt_handler acpi acpi_interrupt_handler
+interrupt_handler acpi acpi_interrupt_handler acpi_sci_irq
 #endif
 GLOBAL(end_interrupt_handlers)
 
 .align PAGE_SIZE
 ENTRY(syscall_exit)
+    SWITCH_STACK
     POPF
 
     /* Save exit code to return value register (AX) */
@@ -162,13 +181,8 @@ ENTRY(syscall_exit)
 END_FUNC(syscall_exit)
 
 ENTRY(terminate_user_task)
-    SWITCH_STACK
-    POPF
-
-    movl $-EFAULT, cpu_regs_ax(%_ASM_SP)
-    RESTORE_ALL_REGS
-
-    ret
+    mov $-EFAULT, %_ASM_SI
+    jmp syscall_exit
 END_FUNC(terminate_user_task)
 
 .align PAGE_SIZE
@@ -184,7 +198,7 @@ ENTRY(enter_usermode)
     mov %_ASM_DX, %gs:(usermode_private)
 
     /* Now switch stack and address space */
-    syscall_to_usermode
+    _to_usermode switch_stack=1
 
     /* SS + SP */
     push $__USER_DS
@@ -202,12 +216,11 @@ ENTRY(enter_usermode)
 END_FUNC(enter_usermode)
 
 ENTRY(syscall_handler_entry)
+    check_syscall_exit
+
     SAVE_ALL_REGS include_ax=0
     syscall_from_usermode
 
-    cmp $SYSCALL_EXIT, %_ASM_AX
-    jz syscall_exit
-
     push %_ASM_CX
     push %r11
 
@@ -224,39 +237,37 @@ ENTRY(syscall_handler_entry)
 END_FUNC(syscall_handler_entry)
 
 ENTRY(sysenter_handler_entry)
+    check_syscall_exit
+
     SAVE_ALL_REGS include_ax=0
-    syscall_from_usermode
+    syscall_from_usermode switch_stack=0
 
     MASK_USER_FLAGS
 
-    cmp $SYSCALL_EXIT, %_ASM_AX
-    jz syscall_exit
-
     mov %_ASM_DI, %_ASM_DX
     mov %r10, %_ASM_CX
     mov %_ASM_AX, %_ASM_DI
     call syscall_handler
 
-    syscall_to_usermode
+    syscall_to_usermode switch_stack=0
     RESTORE_ALL_REGS include_ax=0
 
     SYSEXIT
 END_FUNC(sysenter_handler_entry)
 
 ENTRY(int80_handler_entry)
+    check_syscall_exit
+
     SAVE_ALL_REGS include_ax=0
-    syscall_from_usermode
+    syscall_from_usermode switch_stack=0
 
     MASK_USER_FLAGS
 
-    cmp $SYSCALL_EXIT, %_ASM_AX
-    jz syscall_exit
-
     mov %_ASM_DI, %_ASM_CX
     mov %_ASM_AX, %_ASM_DI
     call syscall_handler
 
-    syscall_to_usermode
+    syscall_to_usermode switch_stack=0
     RESTORE_ALL_REGS include_ax=0
     IRET
 END_FUNC(int80_handler_entry)

arch/x86/traps.c

@@ -39,7 +39,8 @@
 
 #include <mm/vmm.h>
 
-extern void asm_interrupt_handler_uart(void);
+extern void asm_interrupt_handler_uart1(void);
+extern void asm_interrupt_handler_uart2(void);
 extern void asm_interrupt_handler_keyboard(void);
 extern void asm_interrupt_handler_timer(void);
 
@@ -153,15 +154,15 @@ void init_traps(const cpu_t *cpu) {
     set_intr_gate(&percpu->idt[X86_EX_DF],  __KERN_CS, _ul(entry_DF),  GATE_DPL0, GATE_PRESENT, 1);
 #endif
 
-    set_intr_gate(&percpu->idt[COM1_IRQ0_OFFSET], __KERN_CS,
-                  _ul(asm_interrupt_handler_uart), GATE_DPL0, GATE_PRESENT, 0);
-    set_intr_gate(&percpu->idt[COM2_IRQ0_OFFSET], __KERN_CS,
-                  _ul(asm_interrupt_handler_uart), GATE_DPL0, GATE_PRESENT, 0);
-    set_intr_gate(&percpu->idt[PIT_IRQ0_OFFSET], __KERN_CS,
+    set_intr_gate(&percpu->idt[SERIAL_COM1_IRQ], __KERN_CS,
+                  _ul(asm_interrupt_handler_uart1), GATE_DPL0, GATE_PRESENT, 0);
+    set_intr_gate(&percpu->idt[SERIAL_COM2_IRQ], __KERN_CS,
+                  _ul(asm_interrupt_handler_uart2), GATE_DPL0, GATE_PRESENT, 0);
+    set_intr_gate(&percpu->idt[TIMER_IRQ], __KERN_CS,
                   _ul(asm_interrupt_handler_timer), GATE_DPL0, GATE_PRESENT, 0);
-    set_intr_gate(&percpu->idt[KEYBOARD_PORT1_IRQ0_OFFSET], __KERN_CS,
+    set_intr_gate(&percpu->idt[KB_PORT1_IRQ], __KERN_CS,
                   _ul(asm_interrupt_handler_keyboard), GATE_DPL0, GATE_PRESENT, 0);
-    set_intr_gate(&percpu->idt[APIC_TIMER_IRQ_OFFSET], __KERN_CS,
+    set_intr_gate(&percpu->idt[APIC_TIMER_IRQ], __KERN_CS,
                   _ul(asm_interrupt_handler_timer), GATE_DPL0, GATE_PRESENT, 0);
 
     barrier();
@@ -301,7 +302,7 @@ static bool extables_fixup(struct cpu_regs *regs) {
     return false;
 }
 
-void do_exception(struct cpu_regs *regs) {
+void do_exception(cpu_regs_t *regs) {
     static char ec_str[32], panic_str[128];
 
     if (!enter_from_usermode(regs->exc.cs) && extables_fixup(regs))

common/setup.c

@@ -227,9 +227,6 @@ void __noreturn __text_init kernel_start(uint32_t multiboot_magic, unsigned long
 
     init_tasks();
 
-    /* Initialize timers */
-    init_timers(bsp);
-
     /* Try to initialize ACPI (and MADT) */
 #ifndef KTF_ACPICA
     if (init_acpi() < 0) {
@@ -241,11 +238,14 @@ void __noreturn __text_init kernel_start(uint32_t multiboot_magic, unsigned long
             boot_flags.nosmp = true;
     }
 
+    init_ioapic();
+
+    /* Initialize timers */
+    init_timers(bsp);
+
     if (!boot_flags.nosmp)
         init_smp();
 
-    init_ioapic();
-
     init_pci();
 
     /* Initialize console input */

drivers/keyboard.c

@@ -145,7 +145,7 @@ void init_keyboard(const cpu_t *cpu) {
         outb(KEYBOARD_PORT_CMD, KEYBOARD_CMD_WRITE_CONFIGURATION);
         outb(KEYBOARD_PORT_DATA, current_status.config);
 
-        configure_isa_irq(KEYBOARD_PORT1_IRQ, KEYBOARD_PORT1_IRQ0_OFFSET,
+        configure_isa_irq(KEYBOARD_PORT1_IRQ, KEYBOARD_PORT1_IRQ_OFFSET,
                           IOAPIC_DEST_MODE_PHYSICAL, cpu->id);
         outb(KEYBOARD_PORT_CMD, KEYBOARD_CMD_ENABLE_PORT_1);
     }
@@ -155,7 +155,7 @@ void init_keyboard(const cpu_t *cpu) {
         outb(KEYBOARD_PORT_CMD, KEYBOARD_CMD_WRITE_CONFIGURATION);
         outb(KEYBOARD_PORT_DATA, current_status.config);
 
-        configure_isa_irq(KEYBOARD_PORT2_IRQ, KEYBOARD_PORT2_IRQ0_OFFSET,
+        configure_isa_irq(KEYBOARD_PORT2_IRQ, KEYBOARD_PORT2_IRQ_OFFSET,
                           IOAPIC_DEST_MODE_PHYSICAL, cpu->id);
         outb(KEYBOARD_PORT_CMD, KEYBOARD_CMD_ENABLE_PORT_2);
     }

drivers/pit.c

@@ -35,7 +35,7 @@ void init_pit(const cpu_t *cpu) {
     outb(PIT_COMMAND_PORT, PIT_CHANNEL_0 | PIT_ACCESS_MODE_LH | PIT_OP_MODE_RATE);
     outb(PIT_DATA_PORT_CH0, PIT_FREQUENCY & 0xFF);          /* send low byte */
     outb(PIT_DATA_PORT_CH0, (PIT_FREQUENCY & 0xFF00) >> 8); /* send high byte */
-    configure_isa_irq(PIT_IRQ, PIT_IRQ0_OFFSET, IOAPIC_DEST_MODE_PHYSICAL, cpu->id);
+    configure_isa_irq(PIT_IRQ, PIT_IRQ_OFFSET, IOAPIC_DEST_MODE_PHYSICAL, cpu->id);
 }
 
 void pit_disable(void) {