Implement user mode tasks with syscall support

arch/x86/entry.S

@@ -29,8 +29,26 @@
 #include <page.h>
 #include <traps.h>
 
+.macro _handle_usermode cr3
+    testb $0x3, 8(%_ASM_SP)
+    jz 1f /* skip if from kernel mode */
+        SET_CR3 \cr3
+        swapgs
+    1:
+.endm
+
+.macro from_usermode
+    _handle_usermode cr3
+.endm
+
+.macro to_usermode
+    _handle_usermode user_cr3
+.endm
+
 .macro exception_handler sym vec has_error_code
 ENTRY(entry_\sym)
+    from_usermode
+
     .if \has_error_code == 0
         push $0
     .endif
@@ -42,10 +60,14 @@ END_FUNC(entry_\sym)
 
 .macro interrupt_handler sym func
 ENTRY(asm_interrupt_handler_\sym)
+    from_usermode
+
     cld
     SAVE_ALL_REGS
     call \func
     RESTORE_ALL_REGS
+
+    to_usermode
     IRET
 END_FUNC(asm_interrupt_handler_\sym)
 .endm
@@ -87,6 +109,8 @@ ENTRY(handle_exception)
 #else
     add $4, %_ASM_SP
 #endif
+
+    to_usermode
     IRET
 END_FUNC(handle_exception)
 
@@ -97,17 +121,22 @@ interrupt_handler keyboard keyboard_interrupt_handler
 interrupt_handler acpi acpi_interrupt_handler
 #endif
 
-ENTRY(usermode_call_asm)
+ENTRY(enter_usermode)
     /* FIXME: Add 32-bit support */
 
     /* will be restored on entering back in kernel mode */
+    SAVE_ALL_REGS
     PUSHF
 
-    mov %_ASM_SP, %gs:(%rdx)
+    /* Save stack pointer onto per-cpu */
+    mov %_ASM_SP, %gs:(%_ASM_DX)
+
+    /* Move to user stack */
+    mov %_ASM_CX, %_ASM_SP
 
     /* SS + SP */
     push $__USER_DS
-    push %gs:(%rcx)
+    push %_ASM_CX
 
     /* EFLAGS */
     PUSHF
@@ -118,8 +147,9 @@ ENTRY(usermode_call_asm)
     pushq $__USER_CS
     push $usermode_stub
 
+    to_usermode
     IRET
-END_FUNC(usermode_call_asm)
+END_FUNC(enter_usermode)
 
 SECTION(.text.user, "ax", 16)
 ENTRY(usermode_stub)
@@ -130,7 +160,10 @@ ENTRY(usermode_stub)
     xchg %_ASM_DI, %_ASM_SI
     call *%_ASM_SI
 
-    int $X86_RET2KERN_INT
+    /* sys_exit */
+    mov %_ASM_AX, %_ASM_DI
+    xor %_ASM_AX, %_ASM_AX
+    syscall
 END_FUNC(usermode_stub)
 
 SECTION(.data.rmode, "aw", 16)

arch/x86/traps.c

@@ -1,4 +1,5 @@
 /*
+ * Copyright © 2022 Open Source Security, Inc.
  * Copyright © 2020 Amazon.com, Inc. or its affiliates.
  * Copyright © 2014,2015 Citrix Systems Ltd.
  * All Rights Reserved.
@@ -26,28 +27,22 @@
 #include <drivers/keyboard.h>
 #include <drivers/pit.h>
 #include <drivers/serial.h>
+#include <errno.h>
 #include <extables.h>
 #include <ktf.h>
 #include <mm/regions.h>
 #include <percpu.h>
 #include <segment.h>
 #include <symbols.h>
 #include <traps.h>
+#include <usermode.h>
 
 #include <mm/vmm.h>
 
 extern void asm_interrupt_handler_uart(void);
 extern void asm_interrupt_handler_keyboard(void);
 extern void asm_interrupt_handler_timer(void);
 
-static void ret2kern_handler(void) {
-    /* clang-format off */
-    asm volatile("mov %%gs:%[sp], %%" STR(_ASM_SP) "\n"
-                 "POPF \n"
-                 :: [ sp ] "m"(ACCESS_ONCE(PERCPU_VAR(ret2kern_sp))));
-    /* clang-format on */
-}
-
 static void init_tss(percpu_t *percpu) {
 #if defined(__i386__)
     percpu->tss_df.iopb = sizeof(percpu->tss_df);
@@ -153,10 +148,6 @@ void init_traps(const cpu_t *cpu) {
     set_intr_gate(&percpu->idt[X86_EX_DF],  __KERN_CS, _ul(entry_DF),  GATE_DPL0, GATE_PRESENT, 1);
 #endif
 
-    /* User mode return to kernel handler */
-    set_intr_gate(&percpu->idt[X86_RET2KERN_INT], __KERN_CS, _ul(ret2kern_handler), GATE_DPL3, GATE_PRESENT, 0);
-    /* clang-format on */
-
     set_intr_gate(&percpu->idt[COM1_IRQ0_OFFSET], __KERN_CS,
                   _ul(asm_interrupt_handler_uart), GATE_DPL0, GATE_PRESENT, 0);
     set_intr_gate(&percpu->idt[COM2_IRQ0_OFFSET], __KERN_CS,
@@ -174,6 +165,8 @@ void init_traps(const cpu_t *cpu) {
     init_gdt(percpu);
 
     wrmsr(MSR_TSC_AUX, cpu->id);
+
+    init_usermode(percpu);
 }
 
 static void dump_general_regs(const struct cpu_regs *regs) {
@@ -306,7 +299,7 @@ static bool extables_fixup(struct cpu_regs *regs) {
 void do_exception(struct cpu_regs *regs) {
     static char ec_str[32], panic_str[128];
 
-    if (extables_fixup(regs))
+    if (!from_usermode(regs->cs) && extables_fixup(regs))
         return;
 
     dump_regs(regs);
@@ -320,5 +313,11 @@ void do_exception(struct cpu_regs *regs) {
              exception_names[regs->vector], ec_str, regs->cs, regs->_ASM_IP, regs->ss,
              regs->_ASM_SP);
 
+    /* Handle user tasks' exceptions */
+    if (from_usermode(regs->cs)) {
+        printk("Task exception: %s\n", panic_str);
+        goto_syscall_exit(-EFAULT);
+    }
+
     panic(panic_str);
 }

common/kernel.c

@@ -35,14 +35,6 @@
 #include <perfmon/pfmlib.h>
 #endif
 
-extern int usermode_call_asm(user_func_t fn, void *fn_arg, unsigned long ret2kern_sp,
-                             unsigned long user_stack);
-
-int usermode_call(user_func_t fn, void *fn_arg) {
-    return usermode_call_asm(fn, fn_arg, PERCPU_OFFSET(ret2kern_sp),
-                             PERCPU_OFFSET(user_stack));
-}
-
 static void __noreturn echo_loop(void) {
     while (1) {
         io_delay();

common/percpu.c

@@ -49,11 +49,10 @@ percpu_t *get_percpu_page(unsigned int cpu) {
     /* Per CPU page must be identity mapped,
      * because GDT descriptor has 32-bit base.
      */
-    percpu = get_free_page(GFP_IDENT | GFP_KERNEL);
+    percpu = get_free_page(GFP_IDENT | GFP_KERNEL | GFP_USER);
     BUG_ON(!percpu);
 
     percpu->cpu_id = cpu;
-    percpu->user_stack = get_free_page_top(GFP_USER);
 
     list_add(&percpu->list, &percpu_frames);
     return percpu;

common/sched.c

@@ -32,10 +32,12 @@
 #include <setup.h>
 #include <spinlock.h>
 #include <string.h>
+#include <usermode.h>
 
 #include <smp/smp.h>
 
 #include <mm/slab.h>
+#include <mm/vmm.h>
 
 static tid_t next_tid;
 
@@ -96,7 +98,10 @@ static void destroy_task(task_t *task) {
 
     spin_lock(&task->cpu->lock);
     list_unlink(&task->list);
+    if (task->stack)
+        put_page_top(task->stack);
     spin_unlock(&task->cpu->lock);
+
     kfree(task);
 }
 
@@ -111,6 +116,8 @@ static int prepare_task(task_t *task, const char *name, task_func_t func, void *
     task->func = func;
     task->arg = arg;
     task->type = type;
+    if (task->type == TASK_TYPE_USER)
+        task->stack = get_free_page_top(GFP_USER);
     set_task_state(task, TASK_STATE_READY);
     return ESUCCESS;
 }
@@ -178,7 +185,11 @@ static void run_task(task_t *task) {
     printk("CPU[%u]: Running task %s[%u]\n", task->cpu->id, task->name, task->id);
 
     set_task_state(task, TASK_STATE_RUNNING);
-    task->result = task->func(task->arg);
+    if (task->type == TASK_TYPE_USER)
+        task->result = enter_usermode(task->func, task->arg,
+                                      PERCPU_OFFSET(usermode_private), task->stack);
+    else
+        task->result = task->func(task->arg);
     set_task_state(task, TASK_STATE_DONE);
 }