usermode: improve syscall implementation

Main change is a split of the syscall implementation with proper
assembly routines: syscall_handler_entry() and simplified
syscall_exit() and a simplified C-level syscall handler function.
The syscall_exit() is implemented and handled fully in asm.

The syscall() usermode function implements typical syscall ABI,
passing syscall number via AX register and the rest of parameters
via SI, DX, DI, R8 and R9. The assembly entry point rearranges the
registers to fix the C-level handler function API.
Only 5 arguments to the syscall can be provided.
Usermode upon calling syscall() takes are of callee saved registers by
saving/restoring them via usermode stack (only reminder of unused
registers is preserved as the rest is considered implicitely clobbered).

Additionally SYSCALL_PRINTF has been improved to support variadic
number of arguments (it passes the va_list args to the kernel mode).

Signed-off-by: Pawel Wieczorkiewicz <[email protected]>

arch/x86/entry.S

@@ -48,6 +48,18 @@
     _handle_usermode user_cr3
 .endm
 
+.macro syscall_from_usermode
+    SET_CR3 cr3
+    swapgs
+    SWITCH_STACK
+.endm
+
+.macro syscall_to_usermode
+    SWITCH_STACK
+    swapgs
+    SET_CR3 user_cr3
+.endm
+
 .macro exception_handler sym vec has_error_code
 ENTRY(entry_\sym)
     enter_from_usermode
@@ -154,6 +166,16 @@ ENTRY(enter_usermode)
     IRET
 END_FUNC(enter_usermode)
 
+ENTRY(syscall_exit)
+    POPF
+
+    /* Save exit code to return value register (AX) */
+    mov %_ASM_DI, cpu_regs_ax(%_ASM_SP)
+    RESTORE_ALL_REGS
+
+    ret
+END_FUNC(syscall_exit)
+
 ENTRY(terminate_user_task)
     SWITCH_STACK
     POPF
@@ -164,6 +186,28 @@ ENTRY(terminate_user_task)
     ret
 END_FUNC(terminate_user_task)
 
+.align PAGE_SIZE
+ENTRY(syscall_handler_entry)
+    syscall_from_usermode
+
+    cmp $SYSCALL_EXIT, %_ASM_AX
+    jz syscall_exit
+
+    push %_ASM_CX
+    push %r11
+
+    mov %_ASM_DI, %_ASM_CX
+    mov %_ASM_AX, %_ASM_DI
+    call syscall_handler
+
+    pop %r11
+    pop %_ASM_CX
+
+    syscall_to_usermode
+
+    SYSRET
+END_FUNC(syscall_handler_entry)
+
 SECTION(.text.user, "ax", 16)
 ENTRY(usermode_stub)
     /* DI: User function to be called

common/usermode.c

@@ -28,98 +28,19 @@
 #include <processor.h>
 #include <usermode.h>
 
-static inline void syscall_save(void) {
-    /* clang-format off */
-    asm volatile(
-        "push %%" STR(_ASM_CX) "\n"
-        "push %%r11"
-        ::: "memory"
-    );
-    /* clang-format on */
-}
-
-static inline void syscall_restore(void) {
-    /* clang-format off */
-    asm volatile(
-        "pop %%r11\n"
-        "pop %%" STR(_ASM_CX)
-        ::: "memory"
-    );
-    /* clang-format on */
-}
-
-static inline long syscall_return(long return_code) {
-    asm volatile("" ::"a"(return_code));
-    return return_code;
-}
-
-static inline void stack_switch(void) {
-    /* clang-format off */
-    asm volatile(
-        "xchg %%gs:%[private], %%" STR(_ASM_SP) "\n"
-        ::[private] "m"(ACCESS_ONCE(PERCPU_VAR(usermode_private)))
-    );
-    /* clang-format on */
-}
-
-static inline void switch_address_space(const cr3_t *cr3) {
-    /* clang-format off */
-    asm volatile(
-        "push %%" STR(_ASM_AX) "\n"
-        "mov %[cr3], %%" STR(_ASM_AX) "\n"
-        "mov %%" STR(_ASM_AX) ", %%cr3\n"
-        "pop %%" STR(_ASM_AX) "\n"
-        :: [cr3] "m" (*cr3)
-        : STR(_ASM_AX)
-    );
-    /* clang-format on */
-}
-
-static inline void _sys_exit(void) {
-    /* clang-format off */
-    asm volatile (
-        "mov %%" STR(_ASM_DI)", %%gs:%[private]\n"
-        POPF()
-        RESTORE_ALL_REGS()
-        "mov %%gs:%[private], %%" STR(_ASM_AX) "\n"
-        "ret\n"
-        :: [ private ] "m"(ACCESS_ONCE(PERCPU_VAR(usermode_private)))
-        : STR(_ASM_AX)
-    );
-    /* clang-format on */
-}
-
-void __naked syscall_handler(void) {
-    register unsigned long syscall_nr asm(STR(_ASM_AX));
-    register unsigned long param1 asm(STR(_ASM_DI));
-    (void) param1;
-    register unsigned long param2 asm(STR(_ASM_SI));
-    (void) param2;
-    register unsigned long param3 asm(STR(_ASM_BX));
-    (void) param3;
-    register unsigned long param4 asm(STR(_ASM_DX));
-    (void) param4;
-
-    SAVE_CLOBBERED_REGS();
-    switch_address_space(&cr3);
-    swapgs();
-    stack_switch();
-    syscall_save();
-
+long syscall_handler(long syscall_nr, long arg1, long arg2, long arg3, long arg4,
+                     long arg5) {
     switch (syscall_nr) {
     case SYSCALL_EXIT:
-        syscall_restore();
-        _sys_exit();
         UNREACHABLE();
 
     case SYSCALL_PRINTF:
-        printk(_ptr(param1), param2, param3, param4);
-        syscall_return(0);
-        break;
+        vprintk(_ptr(arg1), _ptr(arg2));
+        return 0;
 
     case SYSCALL_MMAP: {
-        void *va = _ptr(param1);
-        unsigned int order = _u(param2);
+        void *va = _ptr(arg1);
+        unsigned int order = _u(arg2);
         frame_t *frame;
 
         frame = get_free_frames(order);
@@ -128,41 +49,32 @@ void __naked syscall_handler(void) {
 
         va = vmap_user(va, frame->mfn, order, L4_PROT_USER, L3_PROT_USER, L2_PROT_USER,
                        L1_PROT_USER);
-        syscall_return(_ul(va));
-    } break;
+        return _ul(va);
+    }
 
     case SYSCALL_MUNMAP: {
-        void *va = _ptr(param1);
-        unsigned int order = _u(param2);
+        void *va = _ptr(arg1);
+        unsigned int order = _u(arg2);
 
         vunmap_user(va, order);
-    } break;
+        return 0;
+    }
 
     default:
         printk("Unknown syscall: %lu\n", syscall_nr);
-        syscall_return(-1L);
-        break;
+        return -1;
     }
-
-    syscall_restore();
-    stack_switch();
-    swapgs();
-    switch_address_space(&user_cr3);
-
-    RESTORE_CLOBBERED_REGS();
-
-    sysret();
 }
 
 static void init_syscall(void) {
     msr_star_t star;
 
-    star.eip = _u(_ul(&syscall_handler));
+    star.eip = _u(_ul(&syscall_handler_entry));
     star.kern_cs = __KERN_CS64;
     star.user_cs = __USER_CS64;
 
     wrmsr(MSR_STAR, star.reg);
-    wrmsr(MSR_LSTAR, _ul(&syscall_handler));
+    wrmsr(MSR_LSTAR, _ul(&syscall_handler_entry));
     /* FIXME: Add compat support */
     wrmsr(MSR_CSTAR, _ul(NULL));
 
@@ -177,67 +89,63 @@ static void init_syscall(void) {
 void init_usermode(percpu_t *percpu) {
     vmap_user_4k(&cr3, virt_to_mfn(&cr3), L1_PROT);
     vmap_user_4k(&enter_usermode, virt_to_mfn(&enter_usermode), L1_PROT);
-    vmap_user_4k(&syscall_handler, virt_to_mfn(&syscall_handler), L1_PROT);
-
+    vmap_user_4k(&syscall_handler_entry, virt_to_mfn(&syscall_handler_entry), L1_PROT);
     init_syscall();
 }
 
-static inline void __user_text sys_exit(unsigned long exit_code) {
-    asm volatile("syscall" ::"A"(SYSCALL_EXIT), "D"(exit_code) : STR(_ASM_CX), "r11");
-}
-
-static inline long __user_text sys_printf(const char *fmt, unsigned long arg1,
-                                          unsigned long arg2, unsigned long arg3) {
-    register unsigned long rax asm(STR(_ASM_AX));
+static inline long __user_text syscall(long syscall_nr, long arg1, long arg2, long arg3,
+                                       long arg4, long arg5) {
+    register long return_code asm(STR(_ASM_AX));
+    register long _arg4 asm("r8") = arg4;
+    register long _arg5 asm("r9") = arg5;
 
     /* clang-format off */
     asm volatile(
-        "syscall"
-        : "=A"(rax)
-        : "0"(SYSCALL_PRINTF), "D"(fmt), "S"(arg1), "b"(arg2), "d"(arg3)
+        SAVE_CALLEE_SAVED_REGS()
+        "syscall\n"
+        RESTORE_CALLEE_SAVED_REGS()
+        : "=a"(return_code)
+        : "0"(syscall_nr), "S"(arg1), "d"(arg2), "D" (arg3), "r"(_arg4), "r"(_arg5)
         : STR(_ASM_CX), "r11"
     );
     /* clang-format on */
 
-    return rax;
+    return return_code;
 }
 
-static inline long __user_text sys_mmap(void *va, unsigned long order) {
-    register unsigned long rax asm(STR(_ASM_AX));
+#define syscall0(nr)                     syscall((nr), 0, 0, 0, 0, 0)
+#define syscall1(nr, a1)                 syscall((nr), (a1), 0, 0, 0, 0)
+#define syscall2(nr, a1, a2)             syscall((nr), (a1), (a2), 0, 0, 0)
+#define syscall3(nr, a1, a2, a3)         syscall((nr), (a1), (a2), (a3), 0, 0)
+#define syscall4(nr, a1, a2, a3, a4)     syscall((nr), (a1), (a2), (a3), (a4), 0)
+#define syscall5(nr, a1, a2, a3, a4, a5) syscall((nr), (a1), (a2), (a3), (a4), (a5))
 
-    /* clang-format off */
-    asm volatile(
-        "syscall"
-        : "=A"(rax)
-        : "0"(SYSCALL_MMAP), "D"(va), "S"(order)
-        : STR(_ASM_CX), "r11"
-    );
-    /* clang-format on */
-
-    return rax;
+static inline void __user_text sys_exit(unsigned long exit_code) {
+    syscall1(SYSCALL_EXIT, exit_code);
 }
 
-static inline long __user_text sys_munmap(void *va, unsigned long order) {
-    register unsigned long rax asm(STR(_ASM_AX));
+static inline long __user_text sys_printf(const char *fmt, va_list args) {
+    return syscall2(SYSCALL_PRINTF, _ul(fmt), _ul(args));
+}
 
-    /* clang-format off */
-    asm volatile(
-        "syscall"
-        ::"A"(SYSCALL_MUNMAP), "D"(va), "S"(order)
-        : STR(_ASM_CX), "r11"
-    );
-    /* clang-format on */
+static inline long __user_text sys_mmap(void *va, unsigned long order) {
+    return syscall2(SYSCALL_MMAP, _ul(va), order);
+}
 
-    return rax;
+static inline long __user_text sys_munmap(void *va, unsigned long order) {
+    return syscall2(SYSCALL_MUNMAP, _ul(va), order);
 }
 
 void __user_text exit(unsigned long exit_code) {
     sys_exit(exit_code);
 }
 
-void __user_text printf(const char *fmt, unsigned long arg1, unsigned long arg2,
-                        unsigned long arg3) {
-    sys_printf(fmt, arg1, arg2, arg3);
+void __user_text printf(const char *fmt, ...) {
+    va_list args;
+
+    va_start(args, fmt);
+    sys_printf(fmt, args);
+    va_end(args);
 }
 
 void *__user_text mmap(void *va, unsigned long order) {