LKMC v3.0

This is a squash commit, the unsquashed development went through many
unstable phases which would break bisects. The unsquashed branch is:
https://github.com/cirosantilli/linux-kernel-module-cheat/tree/v3.0-unsquash

The main improvement of this release was to greatly generalize the testing system.

The key addition was cli_function.py, which allows scripts such as ./run to
be transparently called either from Python or from the command line.

New tests scripts were created using this improved framework: test-baremetal
and test-user-mode.

We were lazy to port some of less important tests to the new setup, TODO's were
added, and we need comes they will be fixed. Getting started is however sacred
as usual and should work.

Other changes include:

-   gem5: update to 7fa4c946386e7207ad5859e8ade0bbfc14000d91

-   run: --tmux-args implies --tmux

-   run: add --userland-args to make userland arguments across QEMU and gem5

    Get rid of --userland-before as a consequence.

-   bring initrd and initramfs back to life

-   build-userland: create --static to make build a bit easier

-   gem5: --gem5-worktree also set --gem5-build-id

-   remove --gem5, use --emulator gem5 everywhere

    Allow passing --emulator multiple times for transparent tests selection
    just like --arch.

-   test-userland: allow selecting just a few tests

-   linux: update to v4.20

-   buildroot: update to 2018.08

    The main motivation for this was to fix the build for Ubuntu 18.10, which
    has glibc 2.28, which broke the 2018.05 build at the m4-host package with:

        #error "Please port gnulib fseeko.c to your platform!

-   getvar --type input

-   failed xen attempt, refactor timer, failed svc attempt, aarch64 use gicv3

-   build-doc: exit 1 on error, add to release testing

-   build: add --apt option to make things easier on other distros

-   build-linux: --no-modules-install

.gitmodules

@@ -21,3 +21,9 @@
 [submodule "submodules/qemu"]
 	path = submodules/qemu
 	url = https://github.com/cirosantilli/qemu
+[submodule "submodules/xen"]
+	path = submodules/xen
+	url = git://xenbits.xen.org/xen.git
+[submodule "submodules/boot-wrapper-aarch64"]
+	path = submodules/boot-wrapper-aarch64
+	url = git://git.kernel.org/pub/scm/linux/kernel/git/mark/boot-wrapper-aarch64.git

baremetal/add.c

@@ -1,13 +1,13 @@
 #include <common.h>
 
 int main(void) {
-	int i, j, k;
-	i = 1;
+    int i, j, k;
+    i = 1;
     /* test-gdb-op1 */
-	j = 2;
+    j = 2;
     /* test-gdb-op2 */
-	k = i + j;
+    k = i + j;
     /* test-gdb-result */
-	if (k != 3)
-		common_assert_fail();
+    if (k != 3)
+        common_assert_fail();
 }

baremetal/arch/aarch64/common_aarch64.h

@@ -0,0 +1,24 @@
+#ifndef COMMON_AARCH64_H
+#define COMMON_AARCH64_H
+
+#include <inttypes.h>
+
+#define SYSREG_READ(type, name) \
+    type sysreg_ ## name ## _read(void) { \
+        type name; \
+        __asm__ __volatile__("mrs %0, " #name : "=r" (name) : : ); \
+        return name; \
+    }
+
+#define SYSREG_WRITE(type, name) \
+    void sysreg_ ## name ## _write(type name) { \
+        __asm__ __volatile__("msr " #name ", %0" : : "r" (name) : ); \
+    }
+
+#define SYSREG_READ_WRITE(name, type) \
+    SYSREG_READ(name, type) \
+    SYSREG_WRITE(name, type)
+
+#define SVC(immediate) __asm__ __volatile__("svc " #immediate : : : )
+
+#endif

baremetal/arch/aarch64/el.c

@@ -4,8 +4,8 @@
 #include <inttypes.h>
 
 int main(void) {
-	register uint64_t x0 __asm__ ("x0");
-	__asm__ ("mrs x0, CurrentEL;" : : : "%x0");
-	printf("%" PRIu64 "\n", x0 >> 2);
-	return 0;
+    uint64_t el;
+    __asm__ ("mrs %0, CurrentEL;" : "=r" (el) : :);
+    printf("%" PRIu64 "\n", el >> 2);
+    return 0;
 }

baremetal/arch/aarch64/multicore.S

@@ -9,6 +9,7 @@ main:
 
     /* Read cpu id into x1.
      * TODO: cores beyond 4th?
+     * Mnemonic: Main Processor ID Register
      */
     mrs x1, mpidr_el1
     ands x1, x1, 3

baremetal/arch/aarch64/svc.c

@@ -0,0 +1,28 @@
+#include <stdio.h>
+#include <inttypes.h>
+
+#include "common_aarch64.h"
+
+/* Masks each of the 4 exception types: Synchronous, System error,
+ * IRQ and FIQ.
+ */
+SYSREG_READ_WRITE(uint32_t, daif)
+
+/* Determines if we use SP0 or SPx. Default: SP0.
+ * See also: https://stackoverflow.com/questions/29393677/armv8-exception-vector-significance-of-el0-sp
+ */
+SYSREG_READ_WRITE(uint32_t, spsel)
+
+/* Jump to this SP if spsel == SPx. */
+SYSREG_READ_WRITE(uint64_t, sp_el1)
+
+int main(void) {
+    printf("daif 0x%" PRIx32 "\n", sysreg_daif_read());
+    printf("spsel 0x%" PRIx32 "\n", sysreg_spsel_read());
+    /* TODO this breaks execution because reading system registers that end
+     * in ELx "trap", leading into an exception on the upper EL.
+     */
+    /*printf("sp_el1 0x%" PRIx64 "\n", sysreg_sp_el1_read());*/
+    /*SVC(0);*/
+    return 0;
+}

baremetal/arch/aarch64/timer.c

@@ -0,0 +1,59 @@
+#include <stdio.h>
+#include <inttypes.h>
+
+#include "common_aarch64.h"
+
+#define CNTV_CTL_ENABLE  (1 << 0)
+#define CNTV_CTL_IMASK   (1 << 1)
+#define CNTV_CTL_ISTATUS (1 << 2)
+
+/* Frequency in Hz. ? */
+SYSREG_READ_WRITE(uint64_t, cntfrq_el0)
+
+/* Current virtual counter value. */
+SYSREG_READ(uint64_t, cntvct_el0)
+
+/* Compare value. See: cntv_ctl_el0_enable. */
+SYSREG_READ_WRITE(uint64_t, cntv_cval_el0)
+
+/* On write, set cntv_cval_el0 = (cntvct_el0 + cntv_tval_el0).
+ * This means that the next interrupt will happen in cntv_tval_el0 cycles.
+ */
+SYSREG_READ_WRITE(uint64_t, cntv_tval_el0)
+
+/* Control register. */
+SYSREG_READ_WRITE(uint32_t, cntv_ctl_el0)
+
+void cntv_ctl_el0_disable(void) {
+    sysreg_cntv_ctl_el0_write(sysreg_cntv_ctl_el0_read() & ~CNTV_CTL_ENABLE);
+}
+
+/* If enabled, when: cntv_ctl > cntv_cval then:
+ *
+ * * if CNTV_CTL_IMASK is clear, raise an interrupt
+ * * set CNTV_CTL_ISTATUS
+ */
+void cntv_ctl_el0_enable(void) {
+    sysreg_cntv_ctl_el0_write(sysreg_cntv_ctl_el0_read() | CNTV_CTL_ENABLE);
+}
+
+int main(void) {
+    /* Initial state. */
+    printf("cntv_ctl_el0 0x%" PRIx32 "\n", sysreg_cntv_ctl_el0_read());
+    printf("cntfrq_el0 0x%" PRIx64 "\n", sysreg_cntfrq_el0_read());
+    printf("cntv_cval_el0 0x%" PRIx64 "\n", sysreg_cntv_cval_el0_read());
+
+    /* Get the counter value many times to watch the time pass. */
+    printf("cntvct_el0 0x%" PRIx64 "\n", sysreg_cntvct_el0_read());
+    printf("cntvct_el0 0x%" PRIx64 "\n", sysreg_cntvct_el0_read());
+    printf("cntvct_el0 0x%" PRIx64 "\n", sysreg_cntvct_el0_read());
+
+#if 0
+    /* TODO crashes gem5. */
+    puts("cntfrq_el0 = 1");
+    sysreg_cntfrq_el0_write(1);
+    printf("cntfrq_el0 0x%" PRIx64 "\n", sysreg_cntfrq_el0_read());
+#endif
+
+    return 0;
+}

baremetal/arch/arm/el.c

@@ -4,8 +4,8 @@
 #include <inttypes.h>
 
 int main(void) {
-	register uint32_t r0 __asm__ ("r0");
-	__asm__ ("mrs r0, CPSR" : : : "%r0");
-	printf("%" PRIu32 "\n", r0 & 0x1F);
-	return 0;
+    uint32_t cpsr;
+    __asm__ ("mrs %0, CPSR" : "=r" (cpsr) : :);
+    printf("%" PRIu32 "\n", cpsr & 0x1F);
+    return 0;
 }

baremetal/exit.c

@@ -2,6 +2,5 @@
 #include <stdlib.h>
 
 int main(void) {
-	exit(0);
+    exit(0);
 }
-

baremetal/interactive/hello.c → baremetal/hello.c

@@ -1,6 +1,6 @@
 #include <stdio.h>
 
 int main(void) {
-	puts("hello");
-	return 0;
+    puts("hello");
+    return 0;
 }

baremetal/interactive/assert_fail.c

@@ -1,6 +1,6 @@
 #include <common.h>
 
 int main(void) {
-	common_assert_fail();
+    common_assert_fail();
 }
 

baremetal/interactive/exit1.c

@@ -2,5 +2,5 @@
 #include <stdlib.h>
 
 int main(void) {
-	exit(1);
+    exit(1);
 }

baremetal/interactive/infinite_loop.c

@@ -0,0 +1,4 @@
+int main(void) {
+    while(1) {}
+    return 0;
+}

baremetal/lib/syscalls.c

@@ -64,24 +64,24 @@ int _write(int file, char *ptr, int len) {
 void _exit(int status) {
 #if defined(GEM5)
 #if defined(__arm__)
-	__asm__ __volatile__ ("mov r0, #0; mov r1, #0; .inst 0xEE000110 | (0x21 << 16);");
+    __asm__ __volatile__ ("mov r0, #0; mov r1, #0; .inst 0xEE000110 | (0x21 << 16);");
 #elif defined(__aarch64__)
-	__asm__ __volatile__ ("mov x0, #0; .inst 0XFF000110 | (0x21 << 16);");
+    __asm__ __volatile__ ("mov x0, #0; .inst 0XFF000110 | (0x21 << 16);");
 #endif
 #else
 #if defined(__arm__)
     __asm__ __volatile__ ("mov r0, #0x18; ldr r1, =#0x20026; svc 0x00123456");
 #elif defined(__aarch64__)
-	/* TODO actually use the exit value here, just for fun. */
+    /* TODO actually use the exit value here, just for fun. */
     __asm__ __volatile__ (
-		"mov x1, #0x26\n" \
-		"movk x1, #2, lsl #16\n" \
-		"str x1, [sp,#0]\n" \
-		"mov x0, #0\n" \
-		"str x0, [sp,#8]\n" \
-		"mov x1, sp\n" \
-		"mov w0, #0x18\n" \
-		"hlt 0xf000\n"
+        "mov x1, #0x26\n" \
+        "movk x1, #2, lsl #16\n" \
+        "str x1, [sp,#0]\n" \
+        "mov x0, #0\n" \
+        "str x0, [sp,#8]\n" \
+        "mov x1, sp\n" \
+        "mov w0, #0x18\n" \
+        "hlt 0xf000\n"
     );
 #endif
 #endif

bench-all

@@ -100,25 +100,25 @@ if "$bench_gem5_build"; then
   common_arch="$default_arch"
   gem5_build_id=bench
   common_gem5_build_dir="$("$getvar" --arch "$common_arch" --gem5-build-id "$gem5_build_id" gem5_build_dir)"
-  common_gem5_src_dir="$("$getvar" --arch "$common_arch" --gem5-build-id "$gem5_build_id" gem5_src_dir)"
+  common_gem5_source_dir="$("$getvar" --arch "$common_arch" --gem5-build-id "$gem5_build_id" gem5_source_dir)"
   results_file="${common_gem5_build_dir}/lkmc-bench-build.txt"
-  git -C "${common_gem5_src_dir}" clean -xdf
+  git -C "${common_gem5_source_dir}" clean -xdf
   rm -f "$results_file"
   "${root_dir}/build-gem5" --arch "$common_arch" --clean --gem5-build-id "$gem5_build_id"
   # TODO understand better: --foreground required otherwise we cannot
   # kill the build with Ctrl+C if something goes wrong, can be minimized to:
   # bash -c "eval 'timeout 5 sleep 3'"
   "${root_dir}/bench-cmd" "timeout --foreground 900 ./build-gem5 --arch '$common_arch' --gem5-build-id '$gem5_build_id'" "$results_file"
   cp "$results_file" "${new_dir}/gem5-bench-build-${common_arch}.txt"
-  git -C "${common_gem5_src_dir}" clean -xdf
+  git -C "${common_gem5_source_dir}" clean -xdf
   "${root_dir}/build-gem5" --arch "$common_arch" --clean --gem5-build-id "$gem5_build_id"
 fi
 
 if "$bench_linux_boot"; then
   cd "${root_dir}"
-  "${root_dir}/build" --all
-  "${root_dir}/bench-boot" --size 3
-  cp "$(${root_dir}/getvar bench_boot)" "$new_dir"
+  "${root_dir}/build" --all-archs all
+  "${root_dir}/test-boot" --size 3
+  cp "$(${root_dir}/getvar test_boot_benchmark_file)" "$new_dir"
 fi
 
 if "$update_repo"; then