diff --git a/arch/x86/boot/head.S b/arch/x86/boot/head.S index 925d86d7..d2825efc 100644 --- a/arch/x86/boot/head.S +++ b/arch/x86/boot/head.S @@ -63,9 +63,9 @@ GLOBAL(_start) mov %eax, %cr3 #if defined (KTF_UNIT_TEST) - SAVE_REGS_32 + SAVE_ALL_REGS32 call prot_to_real - RESTORE_REGS_32 + RESTORE_ALL_REGS32 #endif /* Enable long mode */ diff --git a/arch/x86/entry.S b/arch/x86/entry.S index 63a93f3f..fd11abec 100644 --- a/arch/x86/entry.S +++ b/arch/x86/entry.S @@ -43,9 +43,9 @@ END_FUNC(entry_\sym) .macro interrupt_handler sym func ENTRY(asm_interrupt_handler_\sym) cld - SAVE_REGS + SAVE_ALL_REGS call \func - RESTORE_REGS + RESTORE_ALL_REGS IRET END_FUNC(asm_interrupt_handler_\sym) .endm @@ -75,12 +75,12 @@ exception_handler SE X86_EX_SE 1 ENTRY(handle_exception) cld - SAVE_REGS + SAVE_ALL_REGS mov %_ASM_SP, %_ASM_DI call do_exception - RESTORE_REGS + RESTORE_ALL_REGS #if defined(__x86_64__) add $8, %_ASM_SP diff --git a/arch/x86/traps.c b/arch/x86/traps.c index 5791a6c0..dca1529d 100644 --- a/arch/x86/traps.c +++ b/arch/x86/traps.c @@ -23,20 +23,14 @@ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ -#include -#include #include #include #include #include #include -#include #include #include -#include #include -#include -#include #include #include @@ -83,8 +77,9 @@ static void init_tss(percpu_t *percpu) { percpu->tss.iopb = sizeof(percpu->tss); /* Assign identity mapping of the tss, because GDT has only 32-bit base. */ - percpu->gdt[GDT_TSS].desc = GDT_ENTRY( - DESC_FLAGS(SZ, P, CODE, A), virt_to_paddr(&percpu->tss), sizeof(percpu->tss) - 1); + percpu->gdt[GDT_TSS].desc = + GDT_ENTRY(DESC_FLAGS(GR, SZ, P, DPL0, CODE, A), virt_to_paddr(&percpu->tss), + sizeof(percpu->tss) - 1); #if defined(__x86_64__) percpu->gdt[GDT_TSS + 1].desc = GDT_ENTRY(0x0, 0x0, 0x0); #endif @@ -99,13 +94,15 @@ static void init_gdt(percpu_t *percpu) { percpu->gdt[GDT_KERN_CS32].desc = GDT_ENTRY(DESC_FLAGS(GR, SZ, P, DPL0, S, CODE, R, A), 0x0, 0xfffff); percpu->gdt[GDT_KERN_DS32].desc = GDT_ENTRY(DESC_FLAGS(GR, SZ, P, DPL0, S, DATA, W, A), 0x0, 0xfffff); percpu->gdt[GDT_KERN_CS64].desc = GDT_ENTRY(DESC_FLAGS(GR, L, P, DPL0, S, CODE, R, A), 0x0, 0x00000); + percpu->gdt[GDT_KERN_DS64].desc = GDT_ENTRY(DESC_FLAGS(GR, L, P, DPL0, S, DATA, W, A), 0x0, 0x00000); percpu->gdt[GDT_USER_CS32].desc = GDT_ENTRY(DESC_FLAGS(GR, SZ, P, DPL3, S, CODE, R, A), 0x0, 0xfffff); percpu->gdt[GDT_USER_DS32].desc = GDT_ENTRY(DESC_FLAGS(GR, SZ, P, DPL3, S, DATA, W, A), 0x0, 0xfffff); - percpu->gdt[GDT_USER_CS64].desc = GDT_ENTRY(DESC_FLAGS(GR, L, P, DPL3, S, CODE, R, A), 0x0, 0x00000); + percpu->gdt[GDT_USER_CS64].desc = GDT_ENTRY(DESC_FLAGS(GR, L, P, DPL3, S, CODE, R, A), 0x0, 0xfffff); + percpu->gdt[GDT_USER_DS64].desc = GDT_ENTRY(DESC_FLAGS(GR, L, P, DPL3, S, DATA, W, A), 0x0, 0xfffff); /* Assign identity mapping of the percpu, because GDT has only 32-bit base. */ - percpu->gdt[GDT_PERCPU].desc = GDT_ENTRY(DESC_FLAGS(GR, L, P, DPL3, S, CODE, R, A), virt_to_paddr(percpu), PAGE_SIZE); + percpu->gdt[GDT_PERCPU].desc = GDT_ENTRY(DESC_FLAGS(GR, L, P, DPL0, S, CODE, R, A), virt_to_paddr(percpu), PAGE_SIZE); /* clang-format on */ percpu->gdt_ptr.size = sizeof(percpu->gdt) - 1; @@ -115,6 +112,7 @@ static void init_gdt(percpu_t *percpu) { lgdt(&percpu->gdt_ptr); write_gs(GDT_PERCPU << 3); + wrmsr(MSR_KERNEL_GS_BASE, 0x0); init_tss(percpu); } @@ -220,7 +218,7 @@ static void dump_stack(const struct cpu_regs *regs, unsigned words) { unsigned lines = DUMP_STACK_LINES; printk("STACK[%p]:", sp); - for (unsigned i = 0; i == 0 || (_ul(&sp[i]) % PAGE_SIZE_2M); i++) { + for (unsigned i = 0; i == 0 || (_ul(&sp[i]) % PAGE_SIZE); i++) { if ((i % words) == 0) { if (lines-- == 0) break; diff --git a/include/arch/x86/asm-macros.h b/include/arch/x86/asm-macros.h index 6b946d57..24f9644c 100644 --- a/include/arch/x86/asm-macros.h +++ b/include/arch/x86/asm-macros.h @@ -68,27 +68,7 @@ rep outsb .endm -.macro SAVE_REGS_32 - push %eax - push %ebx - push %ecx - push %edx - push %esi - push %edi - push %ebp -.endm - -.macro RESTORE_REGS_32 - pop %ebp - pop %edi - pop %esi - pop %edx - pop %ecx - pop %ebx - pop %eax -.endm - -.macro SAVE_REGS +.macro SAVE_ALL_REGS32 push %_ASM_AX push %_ASM_BX push %_ASM_CX @@ -96,6 +76,20 @@ push %_ASM_SI push %_ASM_DI push %_ASM_BP +.endm + +.macro RESTORE_ALL_REGS32 + pop %_ASM_BP + pop %_ASM_DI + pop %_ASM_SI + pop %_ASM_DX + pop %_ASM_CX + pop %_ASM_BX + pop %_ASM_AX +.endm + +.macro SAVE_ALL_REGS + SAVE_ALL_REGS32 #if defined(__x86_64__) push %r8 push %r9 @@ -108,7 +102,7 @@ #endif .endm -.macro RESTORE_REGS +.macro RESTORE_ALL_REGS #if defined(__x86_64__) pop %r15 pop %r14 @@ -118,14 +112,30 @@ pop %r10 pop %r9 pop %r8 +#endif + RESTORE_ALL_REGS32 +.endm + +.macro SAVE_CALLEE_SAVED_REGS + push %_ASM_BX + push %_ASM_BP +#if defined(__x86_64__) + push %r12 + push %r13 + push %r14 + push %r15 +#endif +.endm + +.macro RESTORE_CALLEE_SAVED_REGS +#if defined(__x86_64__) + pop %r15 + pop %r14 + pop %r13 + pop %r12 #endif pop %_ASM_BP - pop %_ASM_DI - pop %_ASM_SI - pop %_ASM_DX - pop %_ASM_CX pop %_ASM_BX - pop %_ASM_AX .endm .macro PUSHF @@ -194,6 +204,92 @@ name ## _end: 4:.align 4; /* clang-format on */ +#else + +/* clang-format off */ +#if defined(__x86_64__) +#define SAVE_ALL_REGS64() \ + "push %%r8\n" \ + "push %%r9\n" \ + "push %%r10\n" \ + "push %%r11\n" \ + "push %%r12\n" \ + "push %%r13\n" \ + "push %%r14\n" \ + "push %%r15\n" + +#define RESTORE_ALL_REGS64() \ + "pop %%" STR(r15) "\n" \ + "pop %%" STR(r14) "\n" \ + "pop %%" STR(r13) "\n" \ + "pop %%" STR(r12) "\n" \ + "pop %%" STR(r11) "\n" \ + "pop %%" STR(r10) "\n" \ + "pop %%" STR(r9) "\n" \ + "pop %%" STR(r8) "\n" +#else +#define SAVE_ALL_REGS64() +#define RESTORE_ALL_REGS64() +#endif + +#define SAVE_ALL_REGS() \ + "push %%" STR(_ASM_AX) "\n" \ + "push %%" STR(_ASM_BX) "\n" \ + "push %%" STR(_ASM_CX) "\n" \ + "push %%" STR(_ASM_DX) "\n" \ + "push %%" STR(_ASM_SI) "\n" \ + "push %%" STR(_ASM_DI) "\n" \ + "push %%" STR(_ASM_BP) "\n" \ + SAVE_ALL_REGS64() + +#define RESTORE_ALL_REGS() \ + RESTORE_ALL_REGS64() \ + "pop %%" STR(_ASM_BP) "\n" \ + "pop %%" STR(_ASM_DI) "\n" \ + "pop %%" STR(_ASM_SI) "\n" \ + "pop %%" STR(_ASM_DX) "\n" \ + "pop %%" STR(_ASM_CX) "\n" \ + "pop %%" STR(_ASM_BX) "\n" \ + "pop %%" STR(_ASM_AX) "\n" + +#if defined(__x86_64__) +#define SAVE_CLOBBERED_REGS64() \ + "push %%r8\n" \ + "push %%r9\n" \ + "push %%r10\n" \ + "push %%r11\n" \ + +#define RESTORE_CLOBBERED_REGS64() \ + "pop %%" STR(r11) "\n" \ + "pop %%" STR(r10) "\n" \ + "pop %%" STR(r9) "\n" \ + "pop %%" STR(r8) "\n" +#else +#define SAVE_CLOBBERED_REGS64() +#define RESTORE_CLOBBERED_REGS64() +#endif + +#define SAVE_CLOBBERED_REGS() \ + "push %%" STR(_ASM_CX) "\n" \ + "push %%" STR(_ASM_DX) "\n" \ + "push %%" STR(_ASM_SI) "\n" \ + "push %%" STR(_ASM_DI) "\n" \ + SAVE_CLOBBERED_REGS64() + +#define RESTORE_CLOBBERED_REGS() \ + RESTORE_CLOBBERED_REGS64() \ + "pop %%" STR(_ASM_DI) "\n" \ + "pop %%" STR(_ASM_SI) "\n" \ + "pop %%" STR(_ASM_DX) "\n" \ + "pop %%" STR(_ASM_CX) "\n" \ +/* clang-format on */ + +#if defined(__x86_64__) +#define POPF() "popfq\n" +#else +#define POPF() "popf\n" +#endif + #endif #endif /* KTF_ASM_MACROS_H */ diff --git a/include/arch/x86/processor.h b/include/arch/x86/processor.h index b9a41535..06d5cae5 100644 --- a/include/arch/x86/processor.h +++ b/include/arch/x86/processor.h @@ -26,6 +26,7 @@ #ifndef KTF_PROCESSOR_H #define KTF_PROCESSOR_H +#include #include /* @@ -130,7 +131,7 @@ #define MSR_FS_BASE 0xc0000100 #define MSR_GS_BASE 0xc0000101 -#define MSR_SHADOW_GS_BASE 0xc0000102 +#define MSR_KERNEL_GS_BASE 0xc0000102 #define MSR_TSC_AUX 0xc0000103 diff --git a/include/arch/x86/segment.h b/include/arch/x86/segment.h index 2d00bf95..8ff3b4a8 100644 --- a/include/arch/x86/segment.h +++ b/include/arch/x86/segment.h @@ -33,32 +33,35 @@ #define GDT_KERN_CS32 0x1 #define GDT_KERN_DS32 0x2 #define GDT_KERN_CS64 0x3 +#define GDT_KERN_DS64 0x4 -#define GDT_RMODE_CS16 0x4 -#define GDT_RMODE_DS16 0x5 +#define GDT_RMODE_CS16 0x5 +#define GDT_RMODE_DS16 0x6 -#define GDT_BOOT_TSS 0x4 -#define GDT_BOOT_TSS_DF 0x5 +#define GDT_BOOT_TSS 0x5 +#define GDT_BOOT_TSS_DF 0x6 -#define GDT_USER_CS32 0x4 -#define GDT_USER_DS32 0x5 -#define GDT_USER_CS64 0x6 +#define GDT_USER_CS32 0x5 +#define GDT_USER_DS32 0x6 +#define GDT_USER_CS64 0x7 +#define GDT_USER_DS64 0x8 -#define GDT_TSS 0x7 -#define GDT_TSS_DF 0x8 +#define GDT_TSS 0x9 +#define GDT_TSS_DF 0xa -#define GDT_RMODE_TSS 0x6 -#define GDT_RMODE_TSS_DF 0x7 +#define GDT_RMODE_TSS 0x7 +#define GDT_RMODE_TSS_DF 0x8 -#define GDT_PERCPU 0x9 +#define GDT_PERCPU 0xb -#define NR_BOOT_GDT_ENTRIES 6 -#define NR_RMODE_GDT_ENTRIES 8 -#define NR_GDT_ENTRIES 10 +#define NR_BOOT_GDT_ENTRIES 7 +#define NR_RMODE_GDT_ENTRIES 9 +#define NR_GDT_ENTRIES 12 #define __KERN_CS32 (GDT_KERN_CS32 << 3) #define __KERN_DS32 (GDT_KERN_DS32 << 3) #define __KERN_CS64 (GDT_KERN_CS64 << 3) +#define __KERN_DS64 (GDT_KERN_DS64 << 3) #define __KERN_CS16 (GDT_RMODE_CS16 << 3) #define __KERN_DS16 (GDT_RMODE_DS16 << 3) @@ -66,7 +69,7 @@ #define __USER_CS32 ((GDT_USER_CS32 << 3) | 3) #define __USER_DS32 ((GDT_USER_DS32 << 3) | 3) #define __USER_CS64 ((GDT_USER_CS64 << 3) | 3) -#define __USER_DS64 ((GDT_USER_DS32 << 3) | 3) +#define __USER_DS64 ((GDT_USER_DS64 << 3) | 3) #if defined(__i386__) #define __KERN_CS __KERN_CS32 @@ -76,7 +79,7 @@ #define __USER_DS __USER_DS32 #else #define __KERN_CS __KERN_CS64 -#define __KERN_DS (0) +#define __KERN_DS __KERN_DS64 #define __USER_CS __USER_CS64 #define __USER_DS __USER_DS64 diff --git a/include/lib.h b/include/lib.h index 50d7c3e3..8d20a429 100644 --- a/include/lib.h +++ b/include/lib.h @@ -308,6 +308,18 @@ static inline void flush_tlb(void) { write_cr3(read_cr3()); } static inline void ud2(void) { asm volatile("ud2"); } +static inline void swapgs(void) { asm volatile("swapgs"); } + +static inline void syscall(void) { asm volatile("syscall"); } + +static inline void sysret(void) { +#if defined(__i386__) + asm volatile("sysret"); +#else + asm volatile("sysretq"); +#endif +} + #define BUG() \ do { \ panic("BUG in %s() at line %u\n", __func__, __LINE__); \