From ded8e307ffb7eaadd11668cc64cb22d814e7670e Mon Sep 17 00:00:00 2001 From: Pawel Wieczorkiewicz Date: Fri, 8 Jul 2022 12:30:37 +0200 Subject: [PATCH 01/10] arch: cleanup 32bit and add 64bit SAVE_REGS and RESTORE_REGS macros Signed-off-by: Pawel Wieczorkiewicz --- arch/x86/boot/head.S | 4 +-- arch/x86/entry.S | 8 ++--- include/arch/x86/asm-macros.h | 64 ++++++++++++++++++++--------------- include/arch/x86/processor.h | 1 + 4 files changed, 44 insertions(+), 33 deletions(-) diff --git a/arch/x86/boot/head.S b/arch/x86/boot/head.S index 925d86d7..d2825efc 100644 --- a/arch/x86/boot/head.S +++ b/arch/x86/boot/head.S @@ -63,9 +63,9 @@ GLOBAL(_start) mov %eax, %cr3 #if defined (KTF_UNIT_TEST) - SAVE_REGS_32 + SAVE_ALL_REGS32 call prot_to_real - RESTORE_REGS_32 + RESTORE_ALL_REGS32 #endif /* Enable long mode */ diff --git a/arch/x86/entry.S b/arch/x86/entry.S index 63a93f3f..fd11abec 100644 --- a/arch/x86/entry.S +++ b/arch/x86/entry.S @@ -43,9 +43,9 @@ END_FUNC(entry_\sym) .macro interrupt_handler sym func ENTRY(asm_interrupt_handler_\sym) cld - SAVE_REGS + SAVE_ALL_REGS call \func - RESTORE_REGS + RESTORE_ALL_REGS IRET END_FUNC(asm_interrupt_handler_\sym) .endm @@ -75,12 +75,12 @@ exception_handler SE X86_EX_SE 1 ENTRY(handle_exception) cld - SAVE_REGS + SAVE_ALL_REGS mov %_ASM_SP, %_ASM_DI call do_exception - RESTORE_REGS + RESTORE_ALL_REGS #if defined(__x86_64__) add $8, %_ASM_SP diff --git a/include/arch/x86/asm-macros.h b/include/arch/x86/asm-macros.h index 6b946d57..e98093eb 100644 --- a/include/arch/x86/asm-macros.h +++ b/include/arch/x86/asm-macros.h @@ -68,27 +68,7 @@ rep outsb .endm -.macro SAVE_REGS_32 - push %eax - push %ebx - push %ecx - push %edx - push %esi - push %edi - push %ebp -.endm - -.macro RESTORE_REGS_32 - pop %ebp - pop %edi - pop %esi - pop %edx - pop %ecx - pop %ebx - pop %eax -.endm - -.macro SAVE_REGS +.macro SAVE_ALL_REGS32 push %_ASM_AX push %_ASM_BX push %_ASM_CX @@ -96,6 +76,20 @@ push %_ASM_SI push %_ASM_DI push %_ASM_BP +.endm + +.macro RESTORE_ALL_REGS32 + pop %_ASM_BP + pop %_ASM_DI + pop %_ASM_SI + pop %_ASM_DX + pop %_ASM_CX + pop %_ASM_BX + pop %_ASM_AX +.endm + +.macro SAVE_ALL_REGS + SAVE_ALL_REGS32 #if defined(__x86_64__) push %r8 push %r9 @@ -108,7 +102,7 @@ #endif .endm -.macro RESTORE_REGS +.macro RESTORE_ALL_REGS #if defined(__x86_64__) pop %r15 pop %r14 @@ -118,14 +112,30 @@ pop %r10 pop %r9 pop %r8 +#endif + RESTORE_ALL_REGS32 +.endm + +.macro SAVE_CALLEE_SAVED_REGS + push %_ASM_BX + push %_ASM_BP +#if defined(__x86_64__) + push %r12 + push %r13 + push %r14 + push %r15 +#endif +.endm + +.macro RESTORE_CALLEE_SAVED_REGS +#if defined(__x86_64__) + pop %r15 + pop %r14 + pop %r13 + pop %r12 #endif pop %_ASM_BP - pop %_ASM_DI - pop %_ASM_SI - pop %_ASM_DX - pop %_ASM_CX pop %_ASM_BX - pop %_ASM_AX .endm .macro PUSHF diff --git a/include/arch/x86/processor.h b/include/arch/x86/processor.h index b9a41535..2d9bd949 100644 --- a/include/arch/x86/processor.h +++ b/include/arch/x86/processor.h @@ -26,6 +26,7 @@ #ifndef KTF_PROCESSOR_H #define KTF_PROCESSOR_H +#include #include /* From f50a28f4720fe1c85b7968666b159b88c7e7f2f1 Mon Sep 17 00:00:00 2001 From: Pawel Wieczorkiewicz Date: Fri, 8 Jul 2022 12:51:03 +0200 Subject: [PATCH 02/10] lib: add swapgs, syscall and sysret helper functions Signed-off-by: Pawel Wieczorkiewicz --- include/lib.h | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/include/lib.h b/include/lib.h index 50d7c3e3..8d20a429 100644 --- a/include/lib.h +++ b/include/lib.h @@ -308,6 +308,18 @@ static inline void flush_tlb(void) { write_cr3(read_cr3()); } static inline void ud2(void) { asm volatile("ud2"); } +static inline void swapgs(void) { asm volatile("swapgs"); } + +static inline void syscall(void) { asm volatile("syscall"); } + +static inline void sysret(void) { +#if defined(__i386__) + asm volatile("sysret"); +#else + asm volatile("sysretq"); +#endif +} + #define BUG() \ do { \ panic("BUG in %s() at line %u\n", __func__, __LINE__); \ From ea1fe91cbe7d0861df2daa9caa3b2d4b82c5cc01 Mon Sep 17 00:00:00 2001 From: Pawel Wieczorkiewicz Date: Fri, 8 Jul 2022 12:53:41 +0200 Subject: [PATCH 03/10] arch: dump_stack() should dump at most PAGE_SIZE of stack data Signed-off-by: Pawel Wieczorkiewicz --- arch/x86/traps.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/x86/traps.c b/arch/x86/traps.c index 5791a6c0..cea9802e 100644 --- a/arch/x86/traps.c +++ b/arch/x86/traps.c @@ -220,7 +220,7 @@ static void dump_stack(const struct cpu_regs *regs, unsigned words) { unsigned lines = DUMP_STACK_LINES; printk("STACK[%p]:", sp); - for (unsigned i = 0; i == 0 || (_ul(&sp[i]) % PAGE_SIZE_2M); i++) { + for (unsigned i = 0; i == 0 || (_ul(&sp[i]) % PAGE_SIZE); i++) { if ((i % words) == 0) { if (lines-- == 0) break; From bfb866c05939ffb490f172de5912109aacfcad7c Mon Sep 17 00:00:00 2001 From: Pawel Wieczorkiewicz Date: Fri, 8 Jul 2022 12:59:03 +0200 Subject: [PATCH 04/10] arch, x86: rename MSR_SHADOW_GS_BASE to MSR_KERNEL_GS_BASE Signed-off-by: Pawel Wieczorkiewicz --- include/arch/x86/processor.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/arch/x86/processor.h b/include/arch/x86/processor.h index 2d9bd949..06d5cae5 100644 --- a/include/arch/x86/processor.h +++ b/include/arch/x86/processor.h @@ -131,7 +131,7 @@ #define MSR_FS_BASE 0xc0000100 #define MSR_GS_BASE 0xc0000101 -#define MSR_SHADOW_GS_BASE 0xc0000102 +#define MSR_KERNEL_GS_BASE 0xc0000102 #define MSR_TSC_AUX 0xc0000103 From 62b111f799c76a9e3b861c5645379374a67ca008 Mon Sep 17 00:00:00 2001 From: Pawel Wieczorkiewicz Date: Fri, 8 Jul 2022 13:10:26 +0200 Subject: [PATCH 05/10] arch, segments: Add _KERN_DS64 and _USER_DS64 segment definitions Signed-off-by: Pawel Wieczorkiewicz --- arch/x86/traps.c | 4 +++- include/arch/x86/segment.h | 37 ++++++++++++++++++++----------------- 2 files changed, 23 insertions(+), 18 deletions(-) diff --git a/arch/x86/traps.c b/arch/x86/traps.c index cea9802e..19c5e68f 100644 --- a/arch/x86/traps.c +++ b/arch/x86/traps.c @@ -99,10 +99,12 @@ static void init_gdt(percpu_t *percpu) { percpu->gdt[GDT_KERN_CS32].desc = GDT_ENTRY(DESC_FLAGS(GR, SZ, P, DPL0, S, CODE, R, A), 0x0, 0xfffff); percpu->gdt[GDT_KERN_DS32].desc = GDT_ENTRY(DESC_FLAGS(GR, SZ, P, DPL0, S, DATA, W, A), 0x0, 0xfffff); percpu->gdt[GDT_KERN_CS64].desc = GDT_ENTRY(DESC_FLAGS(GR, L, P, DPL0, S, CODE, R, A), 0x0, 0x00000); + percpu->gdt[GDT_KERN_DS64].desc = GDT_ENTRY(DESC_FLAGS(GR, L, P, DPL0, S, DATA, W, A), 0x0, 0x00000); percpu->gdt[GDT_USER_CS32].desc = GDT_ENTRY(DESC_FLAGS(GR, SZ, P, DPL3, S, CODE, R, A), 0x0, 0xfffff); percpu->gdt[GDT_USER_DS32].desc = GDT_ENTRY(DESC_FLAGS(GR, SZ, P, DPL3, S, DATA, W, A), 0x0, 0xfffff); - percpu->gdt[GDT_USER_CS64].desc = GDT_ENTRY(DESC_FLAGS(GR, L, P, DPL3, S, CODE, R, A), 0x0, 0x00000); + percpu->gdt[GDT_USER_CS64].desc = GDT_ENTRY(DESC_FLAGS(GR, L, P, DPL3, S, CODE, R, A), 0x0, 0xfffff); + percpu->gdt[GDT_USER_DS64].desc = GDT_ENTRY(DESC_FLAGS(GR, L, P, DPL3, S, DATA, W, A), 0x0, 0xfffff); /* Assign identity mapping of the percpu, because GDT has only 32-bit base. */ percpu->gdt[GDT_PERCPU].desc = GDT_ENTRY(DESC_FLAGS(GR, L, P, DPL3, S, CODE, R, A), virt_to_paddr(percpu), PAGE_SIZE); diff --git a/include/arch/x86/segment.h b/include/arch/x86/segment.h index 2d00bf95..8ff3b4a8 100644 --- a/include/arch/x86/segment.h +++ b/include/arch/x86/segment.h @@ -33,32 +33,35 @@ #define GDT_KERN_CS32 0x1 #define GDT_KERN_DS32 0x2 #define GDT_KERN_CS64 0x3 +#define GDT_KERN_DS64 0x4 -#define GDT_RMODE_CS16 0x4 -#define GDT_RMODE_DS16 0x5 +#define GDT_RMODE_CS16 0x5 +#define GDT_RMODE_DS16 0x6 -#define GDT_BOOT_TSS 0x4 -#define GDT_BOOT_TSS_DF 0x5 +#define GDT_BOOT_TSS 0x5 +#define GDT_BOOT_TSS_DF 0x6 -#define GDT_USER_CS32 0x4 -#define GDT_USER_DS32 0x5 -#define GDT_USER_CS64 0x6 +#define GDT_USER_CS32 0x5 +#define GDT_USER_DS32 0x6 +#define GDT_USER_CS64 0x7 +#define GDT_USER_DS64 0x8 -#define GDT_TSS 0x7 -#define GDT_TSS_DF 0x8 +#define GDT_TSS 0x9 +#define GDT_TSS_DF 0xa -#define GDT_RMODE_TSS 0x6 -#define GDT_RMODE_TSS_DF 0x7 +#define GDT_RMODE_TSS 0x7 +#define GDT_RMODE_TSS_DF 0x8 -#define GDT_PERCPU 0x9 +#define GDT_PERCPU 0xb -#define NR_BOOT_GDT_ENTRIES 6 -#define NR_RMODE_GDT_ENTRIES 8 -#define NR_GDT_ENTRIES 10 +#define NR_BOOT_GDT_ENTRIES 7 +#define NR_RMODE_GDT_ENTRIES 9 +#define NR_GDT_ENTRIES 12 #define __KERN_CS32 (GDT_KERN_CS32 << 3) #define __KERN_DS32 (GDT_KERN_DS32 << 3) #define __KERN_CS64 (GDT_KERN_CS64 << 3) +#define __KERN_DS64 (GDT_KERN_DS64 << 3) #define __KERN_CS16 (GDT_RMODE_CS16 << 3) #define __KERN_DS16 (GDT_RMODE_DS16 << 3) @@ -66,7 +69,7 @@ #define __USER_CS32 ((GDT_USER_CS32 << 3) | 3) #define __USER_DS32 ((GDT_USER_DS32 << 3) | 3) #define __USER_CS64 ((GDT_USER_CS64 << 3) | 3) -#define __USER_DS64 ((GDT_USER_DS32 << 3) | 3) +#define __USER_DS64 ((GDT_USER_DS64 << 3) | 3) #if defined(__i386__) #define __KERN_CS __KERN_CS32 @@ -76,7 +79,7 @@ #define __USER_DS __USER_DS32 #else #define __KERN_CS __KERN_CS64 -#define __KERN_DS (0) +#define __KERN_DS __KERN_DS64 #define __USER_CS __USER_CS64 #define __USER_DS __USER_DS64 From 56915b08aa25f905286c4d9b59ec007a27e87c55 Mon Sep 17 00:00:00 2001 From: Pawel Wieczorkiewicz Date: Fri, 8 Jul 2022 13:12:06 +0200 Subject: [PATCH 06/10] arch, percpu: make percpu segment ring0 privilege only Signed-off-by: Pawel Wieczorkiewicz --- arch/x86/traps.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/arch/x86/traps.c b/arch/x86/traps.c index 19c5e68f..b2edfe65 100644 --- a/arch/x86/traps.c +++ b/arch/x86/traps.c @@ -107,7 +107,7 @@ static void init_gdt(percpu_t *percpu) { percpu->gdt[GDT_USER_DS64].desc = GDT_ENTRY(DESC_FLAGS(GR, L, P, DPL3, S, DATA, W, A), 0x0, 0xfffff); /* Assign identity mapping of the percpu, because GDT has only 32-bit base. */ - percpu->gdt[GDT_PERCPU].desc = GDT_ENTRY(DESC_FLAGS(GR, L, P, DPL3, S, CODE, R, A), virt_to_paddr(percpu), PAGE_SIZE); + percpu->gdt[GDT_PERCPU].desc = GDT_ENTRY(DESC_FLAGS(GR, L, P, DPL0, S, CODE, R, A), virt_to_paddr(percpu), PAGE_SIZE); /* clang-format on */ percpu->gdt_ptr.size = sizeof(percpu->gdt) - 1; @@ -117,6 +117,7 @@ static void init_gdt(percpu_t *percpu) { lgdt(&percpu->gdt_ptr); write_gs(GDT_PERCPU << 3); + wrmsr(MSR_KERNEL_GS_BASE, 0x0); init_tss(percpu); } From b9c54aaa23164a6263d6d4950edd10331ce17010 Mon Sep 17 00:00:00 2001 From: Pawel Wieczorkiewicz Date: Fri, 8 Jul 2022 13:13:26 +0200 Subject: [PATCH 07/10] arch, traps: cleanup TSS segment definition and header files Signed-off-by: Pawel Wieczorkiewicz --- arch/x86/traps.c | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/arch/x86/traps.c b/arch/x86/traps.c index b2edfe65..dca1529d 100644 --- a/arch/x86/traps.c +++ b/arch/x86/traps.c @@ -23,20 +23,14 @@ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ -#include -#include #include #include #include #include #include -#include #include #include -#include #include -#include -#include #include #include @@ -83,8 +77,9 @@ static void init_tss(percpu_t *percpu) { percpu->tss.iopb = sizeof(percpu->tss); /* Assign identity mapping of the tss, because GDT has only 32-bit base. */ - percpu->gdt[GDT_TSS].desc = GDT_ENTRY( - DESC_FLAGS(SZ, P, CODE, A), virt_to_paddr(&percpu->tss), sizeof(percpu->tss) - 1); + percpu->gdt[GDT_TSS].desc = + GDT_ENTRY(DESC_FLAGS(GR, SZ, P, DPL0, CODE, A), virt_to_paddr(&percpu->tss), + sizeof(percpu->tss) - 1); #if defined(__x86_64__) percpu->gdt[GDT_TSS + 1].desc = GDT_ENTRY(0x0, 0x0, 0x0); #endif From 6dc58d5202782be201a23b3ff8e451d1d71b964a Mon Sep 17 00:00:00 2001 From: Pawel Wieczorkiewicz Date: Fri, 8 Jul 2022 13:56:51 +0200 Subject: [PATCH 08/10] arch: add POPF() helper macro Signed-off-by: Pawel Wieczorkiewicz --- include/arch/x86/asm-macros.h | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/include/arch/x86/asm-macros.h b/include/arch/x86/asm-macros.h index e98093eb..7dda7967 100644 --- a/include/arch/x86/asm-macros.h +++ b/include/arch/x86/asm-macros.h @@ -204,6 +204,14 @@ name ## _end: 4:.align 4; /* clang-format on */ +#else + +#if defined(__x86_64__) +#define POPF() "popfq\n" +#else +#define POPF() "popf\n" +#endif + #endif #endif /* KTF_ASM_MACROS_H */ From 2d774283ec7af5c2ff6c3ad87dc11e9ec86925e1 Mon Sep 17 00:00:00 2001 From: Pawel Wieczorkiewicz Date: Fri, 8 Jul 2022 13:59:24 +0200 Subject: [PATCH 09/10] arch: add SAVE_ALL_REGS and RESTORE_ALL_REGS for 32/64 bit C code Signed-off-by: Pawel Wieczorkiewicz --- include/arch/x86/asm-macros.h | 47 +++++++++++++++++++++++++++++++++++ 1 file changed, 47 insertions(+) diff --git a/include/arch/x86/asm-macros.h b/include/arch/x86/asm-macros.h index 7dda7967..46ac1e92 100644 --- a/include/arch/x86/asm-macros.h +++ b/include/arch/x86/asm-macros.h @@ -206,6 +206,53 @@ name ## _end: #else +/* clang-format off */ +#if defined(__x86_64__) +#define SAVE_ALL_REGS64() \ + "push %%r8\n" \ + "push %%r9\n" \ + "push %%r10\n" \ + "push %%r11\n" \ + "push %%r12\n" \ + "push %%r13\n" \ + "push %%r14\n" \ + "push %%r15\n" + +#define RESTORE_ALL_REGS64() \ + "pop %%" STR(r15) "\n" \ + "pop %%" STR(r14) "\n" \ + "pop %%" STR(r13) "\n" \ + "pop %%" STR(r12) "\n" \ + "pop %%" STR(r11) "\n" \ + "pop %%" STR(r10) "\n" \ + "pop %%" STR(r9) "\n" \ + "pop %%" STR(r8) "\n" +#else +#define SAVE_ALL_REGS64() +#define RESTORE_ALL_REGS64() +#endif + +#define SAVE_ALL_REGS() \ + "push %%" STR(_ASM_AX) "\n" \ + "push %%" STR(_ASM_BX) "\n" \ + "push %%" STR(_ASM_CX) "\n" \ + "push %%" STR(_ASM_DX) "\n" \ + "push %%" STR(_ASM_SI) "\n" \ + "push %%" STR(_ASM_DI) "\n" \ + "push %%" STR(_ASM_BP) "\n" \ + SAVE_ALL_REGS64() + +#define RESTORE_ALL_REGS() \ + RESTORE_ALL_REGS64() \ + "pop %%" STR(_ASM_BP) "\n" \ + "pop %%" STR(_ASM_DI) "\n" \ + "pop %%" STR(_ASM_SI) "\n" \ + "pop %%" STR(_ASM_DX) "\n" \ + "pop %%" STR(_ASM_CX) "\n" \ + "pop %%" STR(_ASM_BX) "\n" \ + "pop %%" STR(_ASM_AX) "\n" +/* clang-format on */ + #if defined(__x86_64__) #define POPF() "popfq\n" #else From 72c2e676f7b09e767f36b5ddadcdde5acd32cbfe Mon Sep 17 00:00:00 2001 From: Pawel Wieczorkiewicz Date: Fri, 8 Jul 2022 14:00:07 +0200 Subject: [PATCH 10/10] arch: add SAVE/RESTORE macros for clobbered registers (32/64 bits) Signed-off-by: Pawel Wieczorkiewicz --- include/arch/x86/asm-macros.h | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/include/arch/x86/asm-macros.h b/include/arch/x86/asm-macros.h index 46ac1e92..24f9644c 100644 --- a/include/arch/x86/asm-macros.h +++ b/include/arch/x86/asm-macros.h @@ -251,6 +251,37 @@ name ## _end: "pop %%" STR(_ASM_CX) "\n" \ "pop %%" STR(_ASM_BX) "\n" \ "pop %%" STR(_ASM_AX) "\n" + +#if defined(__x86_64__) +#define SAVE_CLOBBERED_REGS64() \ + "push %%r8\n" \ + "push %%r9\n" \ + "push %%r10\n" \ + "push %%r11\n" \ + +#define RESTORE_CLOBBERED_REGS64() \ + "pop %%" STR(r11) "\n" \ + "pop %%" STR(r10) "\n" \ + "pop %%" STR(r9) "\n" \ + "pop %%" STR(r8) "\n" +#else +#define SAVE_CLOBBERED_REGS64() +#define RESTORE_CLOBBERED_REGS64() +#endif + +#define SAVE_CLOBBERED_REGS() \ + "push %%" STR(_ASM_CX) "\n" \ + "push %%" STR(_ASM_DX) "\n" \ + "push %%" STR(_ASM_SI) "\n" \ + "push %%" STR(_ASM_DI) "\n" \ + SAVE_CLOBBERED_REGS64() + +#define RESTORE_CLOBBERED_REGS() \ + RESTORE_CLOBBERED_REGS64() \ + "pop %%" STR(_ASM_DI) "\n" \ + "pop %%" STR(_ASM_SI) "\n" \ + "pop %%" STR(_ASM_DX) "\n" \ + "pop %%" STR(_ASM_CX) "\n" \ /* clang-format on */ #if defined(__x86_64__)