From edda833b695f96f11196ae04602cd1e9a94593f3 Mon Sep 17 00:00:00 2001 From: Pawel Wieczorkiewicz Date: Thu, 31 Aug 2023 14:35:45 +0200 Subject: [PATCH 01/12] asm-offsets: cleanup order of generated header Signed-off-by: Pawel Wieczorkiewicz --- arch/x86/asm-offsets.c | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/arch/x86/asm-offsets.c b/arch/x86/asm-offsets.c index 4f38da74..41028e75 100644 --- a/arch/x86/asm-offsets.c +++ b/arch/x86/asm-offsets.c @@ -34,14 +34,22 @@ void __asm_offset_header(void) { OFFSETOF(usermode_private, percpu_t, usermode_private); - OFFSETOF(cpu_exc_error_code, cpu_exc_t, error_code); OFFSETOF(cpu_exc_vector, cpu_exc_t, vector); + OFFSETOF(cpu_exc_error_code, cpu_exc_t, error_code); OFFSETOF(cpu_exc_ip, cpu_exc_t, _ASM_IP); OFFSETOF(cpu_exc_cs, cpu_exc_t, cs); OFFSETOF(cpu_exc_flags, cpu_exc_t, _ASM_FLAGS); OFFSETOF(cpu_exc_sp, cpu_exc_t, _ASM_SP); OFFSETOF(cpu_exc_ss, cpu_exc_t, ss); + OFFSETOF(cpu_regs_ss, cpu_regs_t, exc.ss); + OFFSETOF(cpu_regs_sp, cpu_regs_t, exc._ASM_SP); + OFFSETOF(cpu_regs_flags, cpu_regs_t, exc._ASM_FLAGS); + OFFSETOF(cpu_regs_cs, cpu_regs_t, exc.cs); + OFFSETOF(cpu_regs_ip, cpu_regs_t, exc._ASM_IP); + OFFSETOF(cpu_regs_error_code, cpu_regs_t, exc.error_code); + OFFSETOF(cpu_regs_vector, cpu_regs_t, exc.vector); + OFFSETOF(cpu_regs_ax, cpu_regs_t, _ASM_AX); OFFSETOF(cpu_regs_bx, cpu_regs_t, _ASM_BX); OFFSETOF(cpu_regs_cx, cpu_regs_t, _ASM_CX); @@ -59,12 +67,4 @@ void __asm_offset_header(void) { OFFSETOF(cpu_regs_r14, cpu_regs_t, r14); OFFSETOF(cpu_regs_r15, cpu_regs_t, r15); #endif - - OFFSETOF(cpu_regs_error_code, cpu_regs_t, exc.error_code); - OFFSETOF(cpu_regs_vector, cpu_regs_t, exc.vector); - OFFSETOF(cpu_regs_ip, cpu_regs_t, exc._ASM_IP); - OFFSETOF(cpu_regs_cs, cpu_regs_t, exc.cs); - OFFSETOF(cpu_regs_flags, cpu_regs_t, exc._ASM_FLAGS); - OFFSETOF(cpu_regs_sp, cpu_regs_t, exc._ASM_SP); - OFFSETOF(cpu_regs_ss, cpu_regs_t, exc.ss); } From ba03b84e8df764e405047ac4ff093dd87f5d3b5b Mon Sep 17 00:00:00 2001 From: Pawel Wieczorkiewicz Date: Thu, 31 Aug 2023 12:06:06 +0200 Subject: [PATCH 02/12] arch,x86: fix cs/ss padding for 32bit on stack irq/exc Signed-off-by: Pawel Wieczorkiewicz --- include/arch/x86/processor.h | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/include/arch/x86/processor.h b/include/arch/x86/processor.h index d0a93541..25360f97 100644 --- a/include/arch/x86/processor.h +++ b/include/arch/x86/processor.h @@ -252,10 +252,12 @@ struct cpu_exc { /* Hardware exception */ x86_reg_t _ASM_IP; - uint16_t cs, _pad_cs[3]; + uint16_t cs; + uint8_t _pad_cs[sizeof(x86_reg_t) - sizeof(uint16_t)]; x86_reg_t _ASM_FLAGS; x86_reg_t _ASM_SP; - uint16_t ss, _pad_ss[3]; + uint16_t ss; + uint8_t _pad_ss[sizeof(x86_reg_t) - sizeof(uint16_t)]; } __packed; typedef struct cpu_exc cpu_exc_t; From a3dbef4bf74a9c02fdb892ca970e93a423796a24 Mon Sep 17 00:00:00 2001 From: Pawel Wieczorkiewicz Date: Wed, 30 Aug 2023 14:35:29 +0200 Subject: [PATCH 03/12] arch,entry: cleanup _(from/to)_usermode macros Add generic _from_usermode and _to_usermode macros that switch pagetable, swap GS and optionally switch stack. Use these macros in syscall_(from/to)_usermode macros directly. Rename enter_(from/to)_usermode macros to cond_(from/to)_usermode macros to highlight the conditional nature of transition. Signed-off-by: Pawel Wieczorkiewicz --- arch/x86/entry.S | 52 ++++++++++++++++++++++++++++-------------------- 1 file changed, 30 insertions(+), 22 deletions(-) diff --git a/arch/x86/entry.S b/arch/x86/entry.S index 22a878d5..bb74cbb5 100644 --- a/arch/x86/entry.S +++ b/arch/x86/entry.S @@ -32,38 +32,46 @@ #include #include -.macro _handle_usermode cr3 +.macro _from_usermode switch_stack=0 + SET_CR3 cr3 + swapgs + .if \switch_stack == 1 + SWITCH_STACK + .endif +.endm + +.macro _to_usermode switch_stack=0 + .if \switch_stack == 1 + SWITCH_STACK + .endif + swapgs + SET_CR3 user_cr3 +.endm + +.macro cond_from_usermode testb $0x3, cpu_exc_cs(%_ASM_SP) jz 1f /* skip if from kernel mode */ - SET_CR3 \cr3 - swapgs + _from_usermode 1: .endm -.macro enter_from_usermode - _handle_usermode cr3 -.endm - -.macro enter_to_usermode - _handle_usermode user_cr3 +.macro cond_to_usermode + testb $0x3, cpu_exc_cs(%_ASM_SP) + jz 1f /* skip if from kernel mode */ + _to_usermode + 1: .endm .macro syscall_from_usermode - SET_CR3 cr3 - swapgs - SWITCH_STACK + _from_usermode .endm .macro syscall_to_usermode - SWITCH_STACK - swapgs - SET_CR3 user_cr3 + _to_usermode .endm .macro exception_handler sym vec has_error_code ENTRY(entry_\sym) - enter_from_usermode - .if \has_error_code == 0 push $0 .endif @@ -74,6 +82,7 @@ ENTRY(entry_\sym) push $\vec #endif + cond_from_usermode jmp handle_exception END_FUNC(entry_\sym) .endm @@ -86,7 +95,7 @@ END_FUNC(entry_\sym) .macro interrupt_handler sym func ENTRY(asm_interrupt_handler_\sym) - enter_from_usermode + _from_usermode MASK_USER_FLAGS @@ -95,7 +104,7 @@ ENTRY(asm_interrupt_handler_\sym) call \func RESTORE_ALL_REGS - enter_to_usermode + _to_usermode IRET END_FUNC(asm_interrupt_handler_\sym) .endm @@ -109,9 +118,8 @@ ENTRY(handle_exception) RESTORE_ALL_REGS + cond_to_usermode add $8, %_ASM_SP - - enter_to_usermode IRET END_FUNC(handle_exception) @@ -184,7 +192,7 @@ ENTRY(enter_usermode) mov %_ASM_DX, %gs:(usermode_private) /* Now switch stack and address space */ - syscall_to_usermode + _to_usermode switch_stack=1 /* SS + SP */ push $__USER_DS From 2acc28a727a3a68cb2ca1780d31b1339d2e33c37 Mon Sep 17 00:00:00 2001 From: Pawel Wieczorkiewicz Date: Fri, 1 Sep 2023 10:41:02 +0200 Subject: [PATCH 04/12] entry: improve stack handling for syscall handlers There is no need to switch stack for sysenter and int80 as the CPU switches to TSS.RSP0. However, it is still necessary for SYSCALL_EXIT since we are terminating the user task. Also, there is no need to save user registers for SYSCALL_EXIT. Signed-off-by: Pawel Wieczorkiewicz --- arch/x86/entry.S | 49 ++++++++++++++++++++++++------------------------ 1 file changed, 25 insertions(+), 24 deletions(-) diff --git a/arch/x86/entry.S b/arch/x86/entry.S index bb74cbb5..01ab1a2c 100644 --- a/arch/x86/entry.S +++ b/arch/x86/entry.S @@ -62,12 +62,12 @@ 1: .endm -.macro syscall_from_usermode - _from_usermode +.macro syscall_from_usermode switch_stack=1 + _from_usermode switch_stack=\switch_stack .endm -.macro syscall_to_usermode - _to_usermode +.macro syscall_to_usermode switch_stack=1 + _to_usermode switch_stack=\switch_stack .endm .macro exception_handler sym vec has_error_code @@ -93,6 +93,14 @@ END_FUNC(entry_\sym) POPF .endm +.macro check_syscall_exit + cmp $SYSCALL_EXIT, %_ASM_AX + jnz 1f + _from_usermode + jmp syscall_exit + 1: +.endm + .macro interrupt_handler sym func ENTRY(asm_interrupt_handler_\sym) _from_usermode @@ -160,6 +168,7 @@ GLOBAL(end_interrupt_handlers) .align PAGE_SIZE ENTRY(syscall_exit) + SWITCH_STACK POPF /* Save exit code to return value register (AX) */ @@ -170,13 +179,8 @@ ENTRY(syscall_exit) END_FUNC(syscall_exit) ENTRY(terminate_user_task) - SWITCH_STACK - POPF - - movl $-EFAULT, cpu_regs_ax(%_ASM_SP) - RESTORE_ALL_REGS - - ret + mov $-EFAULT, %_ASM_SI + jmp syscall_exit END_FUNC(terminate_user_task) .align PAGE_SIZE @@ -210,12 +214,11 @@ ENTRY(enter_usermode) END_FUNC(enter_usermode) ENTRY(syscall_handler_entry) + check_syscall_exit + SAVE_ALL_REGS include_ax=0 syscall_from_usermode - cmp $SYSCALL_EXIT, %_ASM_AX - jz syscall_exit - push %_ASM_CX push %r11 @@ -232,39 +235,37 @@ ENTRY(syscall_handler_entry) END_FUNC(syscall_handler_entry) ENTRY(sysenter_handler_entry) + check_syscall_exit + SAVE_ALL_REGS include_ax=0 - syscall_from_usermode + syscall_from_usermode switch_stack=0 MASK_USER_FLAGS - cmp $SYSCALL_EXIT, %_ASM_AX - jz syscall_exit - mov %_ASM_DI, %_ASM_DX mov %r10, %_ASM_CX mov %_ASM_AX, %_ASM_DI call syscall_handler - syscall_to_usermode + syscall_to_usermode switch_stack=0 RESTORE_ALL_REGS include_ax=0 SYSEXIT END_FUNC(sysenter_handler_entry) ENTRY(int80_handler_entry) + check_syscall_exit + SAVE_ALL_REGS include_ax=0 - syscall_from_usermode + syscall_from_usermode switch_stack=0 MASK_USER_FLAGS - cmp $SYSCALL_EXIT, %_ASM_AX - jz syscall_exit - mov %_ASM_DI, %_ASM_CX mov %_ASM_AX, %_ASM_DI call syscall_handler - syscall_to_usermode + syscall_to_usermode switch_stack=0 RESTORE_ALL_REGS include_ax=0 IRET END_FUNC(int80_handler_entry) From 7cd13ca220439d3ce60267b9984e949796ef8235 Mon Sep 17 00:00:00 2001 From: Pawel Wieczorkiewicz Date: Fri, 1 Sep 2023 10:41:28 +0200 Subject: [PATCH 05/12] unittest: add tests for sysenter and int80 syscall handling Signed-off-by: Pawel Wieczorkiewicz --- tests/unittests.c | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/tests/unittests.c b/tests/unittests.c index 0c5c9fc0..20b13b99 100644 --- a/tests/unittests.c +++ b/tests/unittests.c @@ -80,6 +80,18 @@ static unsigned long __user_text test_user_task_func1(void *arg) { return 0; } +static unsigned long __user_text test_user_task_func1_sysenter(void *arg) { + syscall_mode(SYSCALL_MODE_SYSENTER); + test_user_task_func1(NULL); + return 0; +} + +static unsigned long __user_text test_user_task_func1_int80(void *arg) { + syscall_mode(SYSCALL_MODE_INT80); + test_user_task_func1(NULL); + return 0; +} + static unsigned long __user_text test_user_task_func2(void *arg) { void *va; @@ -165,11 +177,16 @@ int unit_tests(void *_unused) { cpu_freq_expect("Prototyp Amazing Foo One @ 1GHz", 1000000000); cpu_freq_expect("Prototyp Amazing Foo Two @ 1.00GHz", 1000000000); - task_t *task1, *task2, *task_user1, *task_user2, *task_user3; + task_t *task1, *task2, *task_user1, *task_user1_se, *task_user1_int80, *task_user2, + *task_user3; task1 = new_kernel_task("test1", test_kernel_task_func, _ptr(98)); task2 = new_kernel_task("test2", test_kernel_task_func, _ptr(-99)); task_user1 = new_user_task("test1 user", test_user_task_func1, NULL); + task_user1_se = + new_user_task("test1 user sysenter", test_user_task_func1_sysenter, NULL); + task_user1_int80 = + new_user_task("test1 user int80", test_user_task_func1_int80, NULL); task_user2 = new_user_task("test2 user", test_user_task_func2, NULL); task_user3 = new_user_task("test3 user", test_user_task_func3, NULL); @@ -177,6 +194,8 @@ int unit_tests(void *_unused) { schedule_task(task1, get_bsp_cpu()); schedule_task(task2, get_cpu(1)); schedule_task(task_user1, get_bsp_cpu()); + schedule_task(task_user1_se, get_bsp_cpu()); + schedule_task(task_user1_int80, get_bsp_cpu()); schedule_task(task_user2, get_cpu(1)); schedule_task(task_user3, get_bsp_cpu()); From 16182ae2763141f149a54ef6dd39e7dec9a6ac7a Mon Sep 17 00:00:00 2001 From: Pawel Wieczorkiewicz Date: Thu, 31 Aug 2023 12:45:02 +0200 Subject: [PATCH 06/12] entry: always push error_code and vector as machine words By treating both error_code and vector as machines words (either 32bit or 64bit based on built architecture) we simplify handling of the exceptions across architectures. Signed-off-by: Pawel Wieczorkiewicz --- arch/x86/entry.S | 9 +++------ arch/x86/traps.c | 2 +- include/arch/x86/processor.h | 24 +++++++++++------------- include/compiler.h | 2 ++ 4 files changed, 17 insertions(+), 20 deletions(-) diff --git a/arch/x86/entry.S b/arch/x86/entry.S index 01ab1a2c..2b7354a9 100644 --- a/arch/x86/entry.S +++ b/arch/x86/entry.S @@ -75,12 +75,7 @@ ENTRY(entry_\sym) .if \has_error_code == 0 push $0 .endif - -#if defined(__x86_64__) - movl $\vec, cpu_exc_vector(%_ASM_SP) -#else push $\vec -#endif cond_from_usermode jmp handle_exception @@ -127,7 +122,9 @@ ENTRY(handle_exception) RESTORE_ALL_REGS cond_to_usermode - add $8, %_ASM_SP + /* Popping error_code and vector machine words added on entry. */ + /* By using cpu_exc_t offset we make it arch-agnostic. */ + add $cpu_exc_ip, %_ASM_SP IRET END_FUNC(handle_exception) diff --git a/arch/x86/traps.c b/arch/x86/traps.c index 85ca79b5..5e3b1f22 100644 --- a/arch/x86/traps.c +++ b/arch/x86/traps.c @@ -301,7 +301,7 @@ static bool extables_fixup(struct cpu_regs *regs) { return false; } -void do_exception(struct cpu_regs *regs) { +void do_exception(cpu_regs_t *regs) { static char ec_str[32], panic_str[128]; if (!enter_from_usermode(regs->exc.cs) && extables_fixup(regs)) diff --git a/include/arch/x86/processor.h b/include/arch/x86/processor.h index 25360f97..1ede1552 100644 --- a/include/arch/x86/processor.h +++ b/include/arch/x86/processor.h @@ -201,15 +201,19 @@ #include #include +typedef unsigned long x86_reg_t; + union x86_ex_error_code { - uint32_t error_code; + x86_reg_t error_code; struct __packed { - unsigned int E : 1, TLB : 2, index : 13, rsvd_sel : 16; + x86_reg_t E : 1, TLB : 2, index : 13, + rsvd_self : sizeof(x86_reg_t) * BITS_PER_BYTE - 16; }; struct __packed { - unsigned int P : 1, W : 1, U : 1, R : 1, I : 1, rsvd_pfec : 27; + x86_reg_t P : 1, W : 1, U : 1, R : 1, I : 1, + rsvd_pfec : sizeof(x86_reg_t) * BITS_PER_BYTE - 5; }; -}; +} __packed; typedef union x86_ex_error_code x86_ex_error_code_t; /* @@ -239,16 +243,10 @@ extern void entry_SE(void); extern void rmode_exception(void); -#if defined(__x86_64__) -typedef uint64_t x86_reg_t; -#elif defined(__i386__) -typedef uint32_t x86_reg_t; -#endif - struct cpu_exc { - x86_ex_error_code_t error_code; /* Populated by exception entry */ - uint32_t vector; + x86_reg_t vector; + x86_ex_error_code_t error_code; /* Hardware exception */ x86_reg_t _ASM_IP; @@ -301,7 +299,7 @@ typedef uint32_t msr_sysenter_esp_t; typedef uint32_t msr_sysenter_eip_t; #endif -static inline bool has_error_code(uint32_t vector) { +static inline bool has_error_code(x86_reg_t vector) { return !!((1U << vector) & X86_EX_HAS_ERROR_CODE); } diff --git a/include/compiler.h b/include/compiler.h index c3957ef1..176c86c1 100644 --- a/include/compiler.h +++ b/include/compiler.h @@ -26,6 +26,8 @@ #ifndef KTF_COMPILER_H #define KTF_COMPILER_H +#define BITS_PER_BYTE 8 + #define _STR(x) #x #define STR(x) _STR(x) From 0c91785520dda2e02fa4e608948e63b4dd448bf4 Mon Sep 17 00:00:00 2001 From: Pawel Wieczorkiewicz Date: Thu, 31 Aug 2023 13:18:06 +0200 Subject: [PATCH 07/12] arch,traps: cleanup IRQ vectors naming Signed-off-by: Pawel Wieczorkiewicz --- arch/x86/entry.S | 3 ++- arch/x86/traps.c | 17 +++++++++-------- drivers/keyboard.c | 4 ++-- drivers/pit.c | 2 +- drivers/serial.c | 4 ++-- include/acpi_ktf.h | 2 ++ include/arch/x86/apic.h | 1 + include/arch/x86/traps.h | 21 +++++++++++++++++---- include/drivers/hpet.h | 3 ++- include/drivers/keyboard.h | 6 ++++-- include/drivers/pit.h | 5 +++-- include/drivers/serial.h | 6 ++++-- 12 files changed, 49 insertions(+), 25 deletions(-) diff --git a/arch/x86/entry.S b/arch/x86/entry.S index 2b7354a9..c85e86ab 100644 --- a/arch/x86/entry.S +++ b/arch/x86/entry.S @@ -156,7 +156,8 @@ GLOBAL(end_exception_handlers) GLOBAL(interrupt_handlers) interrupt_handler timer timer_interrupt_handler -interrupt_handler uart uart_interrupt_handler +interrupt_handler uart1 uart_interrupt_handler +interrupt_handler uart2 uart_interrupt_handler interrupt_handler keyboard keyboard_interrupt_handler #ifdef KTF_ACPICA interrupt_handler acpi acpi_interrupt_handler diff --git a/arch/x86/traps.c b/arch/x86/traps.c index 5e3b1f22..30b54c54 100644 --- a/arch/x86/traps.c +++ b/arch/x86/traps.c @@ -39,7 +39,8 @@ #include -extern void asm_interrupt_handler_uart(void); +extern void asm_interrupt_handler_uart1(void); +extern void asm_interrupt_handler_uart2(void); extern void asm_interrupt_handler_keyboard(void); extern void asm_interrupt_handler_timer(void); @@ -153,15 +154,15 @@ void init_traps(const cpu_t *cpu) { set_intr_gate(&percpu->idt[X86_EX_DF], __KERN_CS, _ul(entry_DF), GATE_DPL0, GATE_PRESENT, 1); #endif - set_intr_gate(&percpu->idt[COM1_IRQ0_OFFSET], __KERN_CS, - _ul(asm_interrupt_handler_uart), GATE_DPL0, GATE_PRESENT, 0); - set_intr_gate(&percpu->idt[COM2_IRQ0_OFFSET], __KERN_CS, - _ul(asm_interrupt_handler_uart), GATE_DPL0, GATE_PRESENT, 0); - set_intr_gate(&percpu->idt[PIT_IRQ0_OFFSET], __KERN_CS, + set_intr_gate(&percpu->idt[SERIAL_COM1_IRQ], __KERN_CS, + _ul(asm_interrupt_handler_uart1), GATE_DPL0, GATE_PRESENT, 0); + set_intr_gate(&percpu->idt[SERIAL_COM2_IRQ], __KERN_CS, + _ul(asm_interrupt_handler_uart2), GATE_DPL0, GATE_PRESENT, 0); + set_intr_gate(&percpu->idt[TIMER_IRQ], __KERN_CS, _ul(asm_interrupt_handler_timer), GATE_DPL0, GATE_PRESENT, 0); - set_intr_gate(&percpu->idt[KEYBOARD_PORT1_IRQ0_OFFSET], __KERN_CS, + set_intr_gate(&percpu->idt[KB_PORT1_IRQ], __KERN_CS, _ul(asm_interrupt_handler_keyboard), GATE_DPL0, GATE_PRESENT, 0); - set_intr_gate(&percpu->idt[APIC_TIMER_IRQ_OFFSET], __KERN_CS, + set_intr_gate(&percpu->idt[APIC_TIMER_IRQ], __KERN_CS, _ul(asm_interrupt_handler_timer), GATE_DPL0, GATE_PRESENT, 0); barrier(); diff --git a/drivers/keyboard.c b/drivers/keyboard.c index cb7dd720..df9d2eca 100644 --- a/drivers/keyboard.c +++ b/drivers/keyboard.c @@ -145,7 +145,7 @@ void init_keyboard(const cpu_t *cpu) { outb(KEYBOARD_PORT_CMD, KEYBOARD_CMD_WRITE_CONFIGURATION); outb(KEYBOARD_PORT_DATA, current_status.config); - configure_isa_irq(KEYBOARD_PORT1_IRQ, KEYBOARD_PORT1_IRQ0_OFFSET, + configure_isa_irq(KEYBOARD_PORT1_IRQ, KEYBOARD_PORT1_IRQ_OFFSET, IOAPIC_DEST_MODE_PHYSICAL, cpu->id); outb(KEYBOARD_PORT_CMD, KEYBOARD_CMD_ENABLE_PORT_1); } @@ -155,7 +155,7 @@ void init_keyboard(const cpu_t *cpu) { outb(KEYBOARD_PORT_CMD, KEYBOARD_CMD_WRITE_CONFIGURATION); outb(KEYBOARD_PORT_DATA, current_status.config); - configure_isa_irq(KEYBOARD_PORT2_IRQ, KEYBOARD_PORT2_IRQ0_OFFSET, + configure_isa_irq(KEYBOARD_PORT2_IRQ, KEYBOARD_PORT2_IRQ_OFFSET, IOAPIC_DEST_MODE_PHYSICAL, cpu->id); outb(KEYBOARD_PORT_CMD, KEYBOARD_CMD_ENABLE_PORT_2); } diff --git a/drivers/pit.c b/drivers/pit.c index b5328204..cc3c2396 100644 --- a/drivers/pit.c +++ b/drivers/pit.c @@ -35,7 +35,7 @@ void init_pit(const cpu_t *cpu) { outb(PIT_COMMAND_PORT, PIT_CHANNEL_0 | PIT_ACCESS_MODE_LH | PIT_OP_MODE_RATE); outb(PIT_DATA_PORT_CH0, PIT_FREQUENCY & 0xFF); /* send low byte */ outb(PIT_DATA_PORT_CH0, (PIT_FREQUENCY & 0xFF00) >> 8); /* send high byte */ - configure_isa_irq(PIT_IRQ, PIT_IRQ0_OFFSET, IOAPIC_DEST_MODE_PHYSICAL, cpu->id); + configure_isa_irq(PIT_IRQ, PIT_IRQ_OFFSET, IOAPIC_DEST_MODE_PHYSICAL, cpu->id); } void pit_disable(void) { diff --git a/drivers/serial.c b/drivers/serial.c index 12f6f93f..31cf0f28 100644 --- a/drivers/serial.c +++ b/drivers/serial.c @@ -162,8 +162,8 @@ void __text_init init_uart_input(const cpu_t *cpu) { /* Enable IRQ lines */ printk("Enabling serial input\n"); - configure_isa_irq(COM1_IRQ, COM1_IRQ0_OFFSET, IOAPIC_DEST_MODE_PHYSICAL, cpu->id); - configure_isa_irq(COM2_IRQ, COM2_IRQ0_OFFSET, IOAPIC_DEST_MODE_PHYSICAL, cpu->id); + configure_isa_irq(COM1_IRQ, COM1_IRQ_OFFSET, IOAPIC_DEST_MODE_PHYSICAL, cpu->id); + configure_isa_irq(COM2_IRQ, COM2_IRQ_OFFSET, IOAPIC_DEST_MODE_PHYSICAL, cpu->id); } static inline int uart_port_status(io_port_t port) { diff --git a/include/acpi_ktf.h b/include/acpi_ktf.h index 6fd14869..44ece47c 100644 --- a/include/acpi_ktf.h +++ b/include/acpi_ktf.h @@ -275,6 +275,8 @@ extern int init_acpi(void); #include "acpi.h" +#define ACPI_SCI_IRQ 0x9 + typedef void (*acpi_subtable_parser_t)(ACPI_SUBTABLE_HEADER *entry, void *arg); union inti_flags { diff --git a/include/arch/x86/apic.h b/include/arch/x86/apic.h index b3923adc..717bbb40 100644 --- a/include/arch/x86/apic.h +++ b/include/arch/x86/apic.h @@ -32,6 +32,7 @@ #define APIC_IRQ_BASE PIC_IRQ_END_OFFSET #define APIC_TIMER_IRQ_OFFSET (APIC_IRQ_BASE + 0x00) +#define APIC_TIMER_IRQ_VECTOR APIC_TIMER_IRQ_OFFSET #define MSR_X2APIC_REGS 0x800U diff --git a/include/arch/x86/traps.h b/include/arch/x86/traps.h index 52583bcd..431b0604 100644 --- a/include/arch/x86/traps.h +++ b/include/arch/x86/traps.h @@ -23,17 +23,30 @@ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ -#ifndef TRAPS_TRAPS_H -#define TRAPS_TRAPS_H +#ifndef KTF_TRAPS_H +#define KTF_TRAPS_H #include -#define MAX_INT 256 +#define EXC_BASE 0 +#define IRQ_BASE 32 #define SYSCALL_INT 0x80 +#define MAX_INT 256 + #ifndef __ASSEMBLY__ #include +#include +#include +#include + +#define SERIAL_COM1_IRQ COM1_IRQ_VECTOR +#define SERIAL_COM2_IRQ COM2_IRQ_VECTOR +#define TIMER_IRQ PIT_IRQ_VECTOR +#define KB_PORT1_IRQ KEYBOARD_PORT1_IRQ_VECTOR +#define KB_PORT2_IRQ KEYBOARD_PORT2_IRQ_VECTOR +#define APIC_TIMER_IRQ APIC_TIMER_IRQ_VECTOR extern void init_traps(const cpu_t *cpu); extern void init_boot_traps(void); @@ -44,4 +57,4 @@ extern void do_exception(struct cpu_regs *regs); #endif /* __ASSEMBLY__ */ -#endif /* TRAPS_TRAPS_H */ +#endif /* KTF_TRAPS_H */ diff --git a/include/drivers/hpet.h b/include/drivers/hpet.h index d9974606..1a879276 100644 --- a/include/drivers/hpet.h +++ b/include/drivers/hpet.h @@ -37,7 +37,8 @@ #include #define HPET_IRQ PIT_IRQ -#define HPET_IRQ_OFFSET PIT_IRQ0_OFFSET +#define HPET_IRQ_OFFSET PIT_IRQ_OFFSET +#define HPET_IRQ_VECTOR HPET_IRQ_OFFSET #ifndef KTF_ACPICA #define HPET_SIGNATURE (('H') | ('P' << 8) | ('E' << 16) | ('T' << 24)) diff --git a/include/drivers/keyboard.h b/include/drivers/keyboard.h index e9bc8643..18eda214 100644 --- a/include/drivers/keyboard.h +++ b/include/drivers/keyboard.h @@ -33,9 +33,11 @@ #define KEYBOARD_PORT1_IRQ 0x01 #define KEYBOARD_PORT2_IRQ 0x04 /* Second port */ -#define KEYBOARD_PORT1_IRQ0_OFFSET \ +#define KEYBOARD_PORT1_IRQ_OFFSET \ (PIC_IRQ0_OFFSET + KEYBOARD_PORT1_IRQ) /* keyboard first channel irq offset */ -#define KEYBOARD_PORT2_IRQ0_OFFSET (PIC_IRQ0_OFFSET + KEYBOARD_PORT2_IRQ) +#define KEYBOARD_PORT1_IRQ_VECTOR KEYBOARD_PORT1_IRQ_OFFSET +#define KEYBOARD_PORT2_IRQ_OFFSET (PIC_IRQ0_OFFSET + KEYBOARD_PORT2_IRQ) +#define KEYBOARD_PORT2_IRQ_VECTOR KEYBOARD_PORT2_IRQ_OFFSET #define KEYBOARD_STATUS_OUT_FULL 0x01 /* bit set when the keyboard buffer is full */ diff --git a/include/drivers/pit.h b/include/drivers/pit.h index c90b8f55..698238ba 100644 --- a/include/drivers/pit.h +++ b/include/drivers/pit.h @@ -29,8 +29,9 @@ #include #include -#define PIT_IRQ 0x00 /* IRQ line */ -#define PIT_IRQ0_OFFSET (PIC_IRQ0_OFFSET + PIT_IRQ) /* IRQ relative to PIC master */ +#define PIT_IRQ 0x00 /* IRQ line */ +#define PIT_IRQ_OFFSET (PIC_IRQ0_OFFSET + PIT_IRQ) /* IRQ relative to PIC master */ +#define PIT_IRQ_VECTOR PIT_IRQ_OFFSET #define PIT_OUT_FREQUENCY 1193182 /* oscillator output frequency */ #define PIT_RELOAD 1000 /* 1ms */ diff --git a/include/drivers/serial.h b/include/drivers/serial.h index 8e7463e2..f631e07d 100644 --- a/include/drivers/serial.h +++ b/include/drivers/serial.h @@ -176,8 +176,10 @@ enum com_irq { }; typedef enum com_irq com_irq_t; -#define COM1_IRQ0_OFFSET (PIC_IRQ0_OFFSET + COM1_IRQ) -#define COM2_IRQ0_OFFSET (PIC_IRQ0_OFFSET + COM2_IRQ) +#define COM1_IRQ_OFFSET (PIC_IRQ0_OFFSET + COM1_IRQ) +#define COM1_IRQ_VECTOR COM1_IRQ_OFFSET +#define COM2_IRQ_OFFSET (PIC_IRQ0_OFFSET + COM2_IRQ) +#define COM2_IRQ_VECTOR COM2_IRQ_OFFSET #define UART_TXD_REG_OFFSET 0x00 #define UART_RBR_REG_OFFSET 0x00 From 55ad21862787fcbb165f66b4d408109fd600e678 Mon Sep 17 00:00:00 2001 From: Pawel Wieczorkiewicz Date: Thu, 31 Aug 2023 14:41:06 +0200 Subject: [PATCH 08/12] irq: make irq frame layout same as the exception one On IRQ entry always push fake error_code value (always zero) and a corresponding IRQ vector number on the stack to make its layout match the exception stack frame layout. This simplifies the handling of both exceptions and interrupts. Signed-off-by: Pawel Wieczorkiewicz --- arch/x86/asm-offsets.c | 10 ++++++++++ arch/x86/entry.S | 21 +++++++++++++-------- 2 files changed, 23 insertions(+), 8 deletions(-) diff --git a/arch/x86/asm-offsets.c b/arch/x86/asm-offsets.c index 41028e75..2eca586e 100644 --- a/arch/x86/asm-offsets.c +++ b/arch/x86/asm-offsets.c @@ -23,8 +23,10 @@ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ +#include #include #include +#include #define EMIT_DEFINE(s, v) \ asm volatile(".ascii \"@@#define " #s " %0 /* " #v " */@@\"\n" ::"i"(v)) @@ -67,4 +69,12 @@ void __asm_offset_header(void) { OFFSETOF(cpu_regs_r14, cpu_regs_t, r14); OFFSETOF(cpu_regs_r15, cpu_regs_t, r15); #endif + + EMIT_DEFINE(serial_com1_irq, SERIAL_COM1_IRQ); + EMIT_DEFINE(serial_com2_irq, SERIAL_COM2_IRQ); + EMIT_DEFINE(timer_irq, TIMER_IRQ); + EMIT_DEFINE(kb_port1_irq, KB_PORT1_IRQ); + EMIT_DEFINE(kb_port2_irq, KB_PORT2_IRQ); + EMIT_DEFINE(apic_timer_irq, APIC_TIMER_IRQ); + EMIT_DEFINE(acpi_sci_irq, ACPI_SCI_IRQ); } diff --git a/arch/x86/entry.S b/arch/x86/entry.S index c85e86ab..dc4063c7 100644 --- a/arch/x86/entry.S +++ b/arch/x86/entry.S @@ -96,9 +96,11 @@ END_FUNC(entry_\sym) 1: .endm -.macro interrupt_handler sym func +.macro interrupt_handler sym func vec ENTRY(asm_interrupt_handler_\sym) - _from_usermode + push $0 /* fake error_code */ + push $\vec /* IRQ vector */ + cond_from_usermode MASK_USER_FLAGS @@ -107,7 +109,10 @@ ENTRY(asm_interrupt_handler_\sym) call \func RESTORE_ALL_REGS - _to_usermode + cond_to_usermode + /* Popping error_code and vector machine words added on entry. */ + /* By using cpu_exc_t offset we make it arch-agnostic. */ + add $cpu_exc_ip, %_ASM_SP IRET END_FUNC(asm_interrupt_handler_\sym) .endm @@ -155,12 +160,12 @@ exception_handler SE X86_EX_SE 1 GLOBAL(end_exception_handlers) GLOBAL(interrupt_handlers) -interrupt_handler timer timer_interrupt_handler -interrupt_handler uart1 uart_interrupt_handler -interrupt_handler uart2 uart_interrupt_handler -interrupt_handler keyboard keyboard_interrupt_handler +interrupt_handler timer timer_interrupt_handler timer_irq +interrupt_handler uart1 uart_interrupt_handler serial_com1_irq +interrupt_handler uart2 uart_interrupt_handler serial_com2_irq +interrupt_handler keyboard keyboard_interrupt_handler kb_port1_irq #ifdef KTF_ACPICA -interrupt_handler acpi acpi_interrupt_handler +interrupt_handler acpi acpi_interrupt_handler acpi_sci_irq #endif GLOBAL(end_interrupt_handlers) From 35ee4a066864277f2e3202548a727e2ee36dd7a8 Mon Sep 17 00:00:00 2001 From: Pawel Wieczorkiewicz Date: Fri, 1 Sep 2023 11:49:38 +0200 Subject: [PATCH 09/12] arch,entry: mask user flags in exception and irq handling Signed-off-by: Pawel Wieczorkiewicz --- arch/x86/entry.S | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/arch/x86/entry.S b/arch/x86/entry.S index dc4063c7..b5c441e2 100644 --- a/arch/x86/entry.S +++ b/arch/x86/entry.S @@ -32,6 +32,12 @@ #include #include +.macro MASK_USER_FLAGS + PUSHF + andl $~(USERMODE_FLAGS_MASK), (%_ASM_SP) + POPF +.endm + .macro _from_usermode switch_stack=0 SET_CR3 cr3 swapgs @@ -51,6 +57,7 @@ .macro cond_from_usermode testb $0x3, cpu_exc_cs(%_ASM_SP) jz 1f /* skip if from kernel mode */ + MASK_USER_FLAGS _from_usermode 1: .endm @@ -82,12 +89,6 @@ ENTRY(entry_\sym) END_FUNC(entry_\sym) .endm -.macro MASK_USER_FLAGS - PUSHF - andl $~(USERMODE_FLAGS_MASK), (%_ASM_SP) - POPF -.endm - .macro check_syscall_exit cmp $SYSCALL_EXIT, %_ASM_AX jnz 1f @@ -102,8 +103,6 @@ ENTRY(asm_interrupt_handler_\sym) push $\vec /* IRQ vector */ cond_from_usermode - MASK_USER_FLAGS - cld SAVE_ALL_REGS call \func From b3e75f1879c6973029bee9bfafdce13a5d4a61c8 Mon Sep 17 00:00:00 2001 From: Pawel Wieczorkiewicz Date: Wed, 30 Aug 2023 14:34:30 +0200 Subject: [PATCH 10/12] setup: initialize IOAPIC and timers after ACPI The proper order of initialization is: ACPI -> IOAPIC -> Timers. The ACPI provided MADT table provides configuration for IOAPICs and for example HPET. Also, initialized IOAPIC is required by timers to setup their IRQs. Signed-off-by: Pawel Wieczorkiewicz --- common/setup.c | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/common/setup.c b/common/setup.c index dfa58287..97645fea 100644 --- a/common/setup.c +++ b/common/setup.c @@ -227,9 +227,6 @@ void __noreturn __text_init kernel_start(uint32_t multiboot_magic, unsigned long init_tasks(); - /* Initialize timers */ - init_timers(bsp); - /* Try to initialize ACPI (and MADT) */ #ifndef KTF_ACPICA if (init_acpi() < 0) { @@ -241,11 +238,14 @@ void __noreturn __text_init kernel_start(uint32_t multiboot_magic, unsigned long boot_flags.nosmp = true; } + init_ioapic(); + + /* Initialize timers */ + init_timers(bsp); + if (!boot_flags.nosmp) init_smp(); - init_ioapic(); - init_pci(); /* Initialize console input */ From 67018c439b5fad2b619b1ce7b1177de97ed5acd6 Mon Sep 17 00:00:00 2001 From: Pawel Wieczorkiewicz Date: Wed, 30 Aug 2023 16:10:42 +0200 Subject: [PATCH 11/12] grub,unittest: enable timers for unit testing Signed-off-by: Pawel Wieczorkiewicz --- grub/grub-test.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/grub/grub-test.cfg b/grub/grub-test.cfg index bf65f9c0..dd860e1e 100644 --- a/grub/grub-test.cfg +++ b/grub/grub-test.cfg @@ -10,6 +10,6 @@ terminal_input --append serial terminal_output --append serial menuentry "kernel64" { - multiboot2 /boot/kernel64.bin.xz poweroff=1 com1=0x3f8,115200,8,n,1 integer=42 boolean=1 string=foo badstring=toolong booleantwo tests=unit_tests + multiboot2 /boot/kernel64.bin.xz poweroff=1 com1=0x3f8,115200,8,n,1 pit=1 hpet=1 apic_timer=1 integer=42 boolean=1 string=foo badstring=toolong booleantwo tests=unit_tests boot } From 0b85fbe0b0234177d4c45d7e532a13eb4ae0ea91 Mon Sep 17 00:00:00 2001 From: Pawel Wieczorkiewicz Date: Fri, 1 Sep 2023 15:58:53 +0200 Subject: [PATCH 12/12] entry: drop unnecessary CLD from exc./irq handlers Signed-off-by: Pawel Wieczorkiewicz --- arch/x86/entry.S | 2 -- 1 file changed, 2 deletions(-) diff --git a/arch/x86/entry.S b/arch/x86/entry.S index b5c441e2..5432716b 100644 --- a/arch/x86/entry.S +++ b/arch/x86/entry.S @@ -103,7 +103,6 @@ ENTRY(asm_interrupt_handler_\sym) push $\vec /* IRQ vector */ cond_from_usermode - cld SAVE_ALL_REGS call \func RESTORE_ALL_REGS @@ -117,7 +116,6 @@ END_FUNC(asm_interrupt_handler_\sym) .endm ENTRY(handle_exception) - cld SAVE_ALL_REGS mov %_ASM_SP, %_ASM_DI