support keystone on CVA6 (#396)

The pull request contains changes to support running keystone on CVA6 with Genesys 2 Board. 1. Support both CVA6 64bit and 32bit 2. Add dedicated patches for CVA6 3. Add dedicated configuration for CVA6 4. Update CVA6 documentation On top of that, this pull request also 1. Rearrange the Makefile to automatically fetch the configuration based on the KEYSTONE_PLATFORM 2. Fix issue with Keystone 32 Bit on QEMU Several notes in this Pull request 1. This PR does not include Root of Trust in CVA6. Please see the more details in the documentation 2. There is still runtime error in 32 bit but the hello-native can run till then end. possibly due to CVA6 32 does not support D extension Let me know if you have any comment --------- Co-authored-by: Gregor Haas <[email protected]>
keystone-enclave · Jan 9, 2024 · 26e242f · 26e242f
1 parent ae7b689
commit 26e242f
Show file tree

Hide file tree

Showing 46 changed files with 4,866 additions and 524 deletions.
diff --git a/.github/workflows/build-runtime.yml b/.github/workflows/build-runtime.yml
@@ -7,11 +7,11 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        platform: [generic, unmatched]
+        platform: [generic, hifive_unmatched, cva6]
         bits: [32, 64]
         exclude:
           # unmatched is not 32 bit
-          - platform: unmatched
+          - platform: hifive_unmatched
             bits: 32
 
     steps:

diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
@@ -14,11 +14,11 @@ jobs:
     # platform that we support here
     strategy:
       matrix:
-        platform: [generic, unmatched]
+        platform: [generic, hifive_unmatched, cva6]
         bits: [32, 64]
         exclude:
           # unmatched is not 32 bit
-          - platform: unmatched
+          - platform: hifive_unmatched
             bits: 32
 
     steps:

diff --git a/.gitmodules b/.gitmodules
@@ -4,3 +4,6 @@
 [submodule "runtime/test/cmocka"]
         path = runtime/test/cmocka
         url = https://gitlab.com/cmocka/cmocka
+[submodule "overlays/keystone/board/cva6/cva6-sdk"]
+	path = overlays/keystone/board/cva6/cva6-sdk
+	url = https://github.com/openhwgroup/cva6-sdk
diff --git a/CVA6_INSTALL.md b/CVA6_INSTALL.md
diff --git a/Makefile b/Makefile
@@ -23,17 +23,17 @@ export BUILDDIR                 ?= $(KEYSTONE)/build-$(KEYSTONE_PLATFORM)$(KEYST
 export BUILDROOT_OVERLAYDIR     ?= $(BUILDDIR)/overlay
 export BUILDROOT_BUILDDIR       ?= $(BUILDDIR)/buildroot.build
 
+
+# options: generic, cva6, hifive_unmatched
 export KEYSTONE_PLATFORM        ?= generic
 export KEYSTONE_BITS            ?= 64
 
 include mkutils/args.mk
 include mkutils/log.mk
 
-BUILDROOT_CONFIGFILE    ?= qemu_riscv$(KEYSTONE_BITS)_virt_defconfig
+BUILDROOT_CONFIGFILE    ?= riscv$(KEYSTONE_BITS)_$(KEYSTONE_PLATFORM)_defconfig
 ifeq ($(KEYSTONE_PLATFORM),mpfs)
 	EXTERNALS += microchip
-else ifeq ($(KEYSTONE_PLATFORM),unmatched)
-    BUILDROOT_CONFIGFILE = riscv64_hifive_unmatched_defconfig
 endif
 
 # Highest priority external
@@ -76,7 +76,7 @@ $(BUILDROOT_OVERLAYDIR)/.done: $(BUILDROOT_OVERLAYDIR)
 	$(call log,info,Setting up overlay)
 	mkdir -p $(BUILDROOT_OVERLAYDIR)/root/.ssh
 	ssh-keygen -C 'root@keystone' -t rsa -f $(BUILDROOT_OVERLAYDIR)/root/.ssh/id-rsa -N ''
-	cp $(BUILDROOT_OVERLAYDIR)/root/.ssh/{id-rsa.pub,authorized_keys}
+	cp -f $(BUILDROOT_OVERLAYDIR)/root/.ssh/{id-rsa.pub,authorized_keys} 
 	touch $@
 
 # Main build target for buildroot. The specific subtarget to build can be overriden

diff --git a/docs/source/Getting-Started/Running-Keystone-on-CVA6.rst b/docs/source/Getting-Started/Running-Keystone-on-CVA6.rst
@@ -1,94 +1,196 @@
-CVA6 Hardware Deployment
+Running Keystone on CVA6
 ===========================
 
-Building Keystone
-----------------------------------------
+`CVA6 <https://github.com/openhwgroup/cva6/>`_. is a 6-stage, single-issue, in-order CPU which implements the 64-bit RISC-V instruction set. 
 
-Building for the CVA6 is straight-forward.
-First, clone the Keystone repository in the **manager instance**.
+Requirement 
+-----------
+Keystone has been ported so it can run on CVA6 Platform on Genesys 2 Board. It is tested on version `797f0a9 <https://github.com/openhwgroup/cva6/commit/f301d6967517336a21a58c9b8a00ea3186906c01/>`_ for both 64 bit and 32 bit. You need to synthesize cva6 and flash the it Genesys2 Board. 
 
-::
+Root of Trust
+#############
+To enable the Root of Trust for attestation report, you need to modify the CVA6 `bootrom <https://github.com/openhwgroup/cva6/tree/master/corev_apu/fpga/src/bootrom/src/>`_ code to perform TCB measurement and SM key generation. This modification is not archived in CVA6 repository. You can imitate keystone QEMU bootrom to implement the CVA6 RoT
 
-  git clone https://github.com/keystone-enclave/keystone
 
-Follow :doc:`QEMU-Setup-Repository` to setup the repository.
+Floating Point Support
+######################
 
-After you setup the repository, you can run the following commands to build Keystone.
+CVA6 32 bit only support F extension for floating point. Additionally, up to glibc version 2.38, F extension is not yet supported. because of that, you may not be able to use floating point or run application that requires floating point.
 
-::
-  export BUILD_ROOT=<absolute path to build directory>
-  mkdir <build directory>
-  cd <build directory>
-  cmake .. -Dcva6=y
-  make
-  make image
+Building Keystone
+-----------------
 
-CMake with the flag ``-Dcva6=y`` will automatically generate Makefiles to build
-CVA6-compatible Linux and SM. The build will forcibly use initramfs for a simpler deployment.
+To build the keystone, you need to run this command
 
-Once you have built the image, you will see ``sm.build/platform/generic/firmware/fw_payload.bin`` 
-under your build directory. This is the file that you want to reflash the SD card with.
+.. code-block:: bash
 
-You can also boot QEMU machine with the image using ``./scripts/run-qemu.sh``.
+    KEYSTONE_PLATFORM=cva6 make
 
-Setting up the CVA6
----------------------
+or, if you want to build the 32 bit version of keystone to run on CV32A6
 
-FPGA Emulation / Software Simulation
-####################################
+.. code-block:: bash 
+  
+    KEYSTONE_PLATFORM=cva6 KEYSTONE_BITS=32 make
 
-Keystone can either be simulated (https://github.com/openhwgroup/cva6#memory-preloading) or FPGA emulated (https://github.com/openhwgroup/cva6#corev-apu-fpga-emulation) on the Xilinx Genesys 2 board 
-(https://digilent.com/reference/programmable-logic/genesys-2/reference-manual)
 
+Setting up the CVA6
+-------------------
 
-Load Linux Image
+Flash SD Card
 ################
 
-The CVA6 build process generates a fw_payload.bin in
-``<build directory>/sm.build/platform/generic/firmware/fw_payload.bin``. Flash this to the Linux partition on the SD
-card. (Note that this is a relocated version of the OpenSBI binary used for QEMU)
-
-
-Example loading script
-######################
-
-This is an example of a script to load the FSBL and OpenSBI into a card
-for use on the CVA6. Be careful as this will repartition the target disk!
-
-You only need to reprogram the FSBL when modifying the first-stage
-bootloader itself. (Likely never)
-
-
-The CVA6 image uses a simplified structure due to bootloader ROM integration on the device:
-:: code-block:: bash
-#!/bin/bash
-
-set -e
+To flash the SD card, you need to run this command
 
-# format disk
-sgdisk --clear --new=1:2048:67583 --new=2 --typecode=1:3000 --typecode=2:8300 /dev/sdc
+.. code-block:: bash
 
-# flash image
-dd if=sm.build/platform/generic/firmware/fw_payload.bin  of=/dev/sdc1 status=progress oflag=sync bs=1M
+    sudo KEYSTONE_PLATFORM=cva6 SD_DEVICE=/dev/sdb make flash
 
+or, for the 32 bit version
 
-Running on the CVA6
----------------------
+.. code-block:: bash
+   
+    sudo KEYSTONE_PLATFORM=cva6 KEYSTONE_BITS=32 SD_DEVICE=/dev/sdb make flash
 
-The needed driver, bins, etc are included in the buildroot image.
-You can always use buildroot overlay to add more files to the base image.
 
-Generally, we also suggest ``scp`` ing the files to the board after boot.
+make sure that the SD_DEVICE is set to the correct SD Card device name on your machine
 
 
 Run binaries
-#########################
+############
 
 Insert the Keystone driver, and run whatever test binaries you wish.
 
 Example
 
-::
+.. code-block:: bash 
+
+    insmod keystone-driver.ko
+    ./tests.ke
+
+Known Issues
+------------
+
+32 Bit Error
+############
+
+On 32 bit target, during startup, the linux log display this error
+
+.. code-block:: bash
+
+    Starting network: [   38.002974] lowrisc-eth 30000000.lowrisc-eth eth0: Current addr  0[   38.011773] lowrisc-eth 30000000.lowrisc-eth eth0: Expected add0  [   38.020196] ------------[ cut here ]------------
+  
+    [   38.026528] netdevice: eth0: Incorrect netdev->dev_addr
+    [   38.034447] WARNING: CPU: 0 PID: 99 at net/core/dev_addr_lists.
+    [   38.041518] Modules linked in:
+    [   38.045439] CPU: 0 PID: 99 Comm: ip Not tainted 6.1.32 #2
+    [   38.051668] Hardware name: eth,ariane-bare (DT)
+    [   38.056825] epc : dev_addr_check+0xa0/0xf4
+    [   38.061851]  ra : dev_addr_check+0xa0/0xf4
+    [   38.066842] epc : c03709bc ra : c03709bc sp : c23c3d80
+    [   38.072723]  gp : c1c90a60 tp : c22f2f40 t0 : 00000000
+    [   38.078577]  t1 : ffffffff t2 : 00000000 s0 : c23c3da0
+    [   38.084431]  s1 : c215c800 a0 : 0000002b a1 : 3fffefff
+    [   38.090270]  a2 : 00000000 a3 : fffffffe a4 : 00000000
+    [   38.096093]  a5 : 00000000 a6 : c1c62f88 a7 : 00000001
+    [   38.101924]  s2 : c215c800 s3 : c22aedf4 s4 : 00000000
+    [   38.107763]  s5 : 00000000 s6 : c215c97c s7 : 00000000
+    [   38.113592]  s8 : c21bac00 s9 : 00000000 s10: 9d430e68
+    [   38.119428]  s11: 00000008 t3 : 00000000 t4 : 00000000
+    [   38.125249]  t5 : 00000001 t6 : 00000000
+    [   38.136288] [<c03709bc>] dev_addr_check+0xa0/0xf4use: 00000003
+    [   38.141970] [<c036dc3a>] __dev_open+0x2c/0x12c
+    [   38.147345] [<c036e06c>] __dev_change_flags+0x146/0x1b0
+    [   38.153558] [<c036e0f2>] dev_change_flags+0x1c/0x4c
+    [   38.159369] [<c03f2934>] devinet_ioctl+0x1bc/0x4dc
+    [   38.165230] [<c03f4174>] inet_ioctl+0x104/0x136
+    [   38.170637] [<c034a648>] sock_ioctl+0x22a/0x26e
+    [   38.176201] [<c00df3de>] sys_ioctl+0x6e/0x7c
+    [   38.181487] [<c0003182>] ret_from_syscall+0x0/0x2
+    [   38.187090] ---[ end trace 0000000000000000 ]---
+    [   38.210424] Open device, request interrupt 3
+    udhcpc: started, v1.36.0
+
+And when you run the enclave application, such as hello-native, you will see the error log as below
+
+.. code-block:: bash
+
+  # /usr/share/keystone/examples/hello-native.ke
+  /dev/console
+  Verifying archive integrity... MD5 checksums are OK. All good.
+  Uncompressing Keystone Enclave Package
+  [ 2334.409841] rcu: INFO: rcu_sched self-detected stall on CPU
+  [ 2334.416433] rcu:     0-...!: (2100 ticks this GP) idle=a6e4/1/0x402
+  [ 2334.423348]    (t=2100 jiffies g=9217 q=24 ncpus=1)
+  [ 2334.428853] rcu: rcu_sched kthread timer wakeup didn't happen f 
+  [ 2334.435566] rcu:  Possible timer handling issue on cpu=0 timer-
+  [ 2334.442017] rcu: rcu_sched kthread starved for 2100 jiffies! g9T
+  [ 2334.448709] rcu:     Unless rcu_sched kthread gets sufficient CPU e
+  [ 2334.455251] rcu: RCU grace-period kthread stack dump:
+  [ 2334.460899] task:rcu_sched       state:I stack:0     pid:12    0
+  [ 2334.467886] Call Trace:
+  [ 2334.470930] [<c04bbb1e>] __schedule+0x1ca/0x46c
+  [ 2334.476485] [<c04bbdfa>] schedule+0x3a/0xba
+  [ 2334.481575] [<c04c0418>] schedule_timeout+0x62/0xc4
+  [ 2334.487504] [<c00589a0>] rcu_gp_fqs_loop+0x238/0x2a0
+  [ 2334.493394] [<c005a32a>] rcu_gp_kthread+0x5a/0xd4
+  [ 2334.498992] [<c0024c80>] kthread+0x8c/0x98
+  [ 2334.504146] [<c0003190>] ret_from_exception+0x0/0x16
+  [ 2334.510048] rcu: Stack dump where RCU GP kthread last ran:
+  [ 2334.516218] CPU: 0 PID: 190 Comm: hello-native-ru Tainted: G   3
+  [ 2334.522896] Hardware name: eth,ariane-bare (DT)
+  [ 2334.528052] epc : __do_softirq+0xb8/0x25a
+  [ 2334.533035]  ra : irq_exit+0x8c/0xbc
+  [ 2334.537500] epc : c04c1660 ra : c000f266 sp : c23b7b70
+  [ 2334.543396]  gp : c1c90a60 tp : c0810000 t0 : 00046000
+  [ 2334.549285]  t1 : 00002382 t2 : c0000000 s0 : c23b7bd0
+  [ 2334.555163]  s1 : c1cbdd58 a0 : 00000000 a1 : c104fcc0
+  [ 2334.561036]  a2 : 00030f7d a3 : 00030f7e a4 : c7afacc0
+  [ 2334.566906]  a5 : c104fcc0 a6 : 70a61351 a7 : 2c010000
+  [ 2334.572787]  s2 : c1cbdd58 s3 : c23b7c00 s4 : 00000000
+  [ 2334.578657]  s5 : c1c05080 s6 : c104e1e0 s7 : c104e170
+  [ 2334.584537]  s8 : 95709030 s9 : 00000000 s10: c1050000
+  [ 2334.590404]  s11: 00000008 t3 : 000000a1 t4 : 00000014
+  [ 2334.596258]  t5 : a402303c t6 : a4023044
+  [ 2334.607380] [<c04c1660>] __do_softirq+0xb8/0x25aause: 80000005
+  [ 2334.613023] [<c000f266>] irq_exit+0x8c/0xbc
+  [ 2334.618117] [<c04bafb2>] generic_handle_arch_irq+0x64/0x74
+  [ 2334.624537] [<c0003190>] ret_from_exception+0x0/0x16
+  [ 2334.630814] CPU: 0 PID: 190 Comm: hello-native-ru Tainted: G   2
+  [ 2334.637579] Hardware name: eth,ariane-bare (DT)
+  [ 2334.642798] epc : __do_softirq+0xb8/0x25a
+  [ 2334.647783]  ra : irq_exit+0x8c/0xbc
+  [ 2334.652209] epc : c04c1660 ra : c000f266 sp : c23b7b70
+  [ 2334.658114]  gp : c1c90a60 tp : c0810000 t0 : 00046000
+  [ 2334.664003]  t1 : 00002382 t2 : c0000000 s0 : c23b7bd0
+  [ 2334.669881]  s1 : c1cbdd58 a0 : 00000000 a1 : c104fcc0
+  [ 2334.675752]  a2 : 00030f7d a3 : 00030f7e a4 : c7afacc0
+  [ 2334.681625]  a5 : c104fcc0 a6 : 70a61351 a7 : 2c010000
+  [ 2334.687493]  s2 : c1cbdd58 s3 : c23b7c00 s4 : 00000000
+  [ 2334.693356]  s5 : c1c05080 s6 : c104e1e0 s7 : c104e170
+  [ 2334.699233]  s8 : 95709030 s9 : 00000000 s10: c1050000
+  [ 2334.705097]  s11: 00000008 t3 : 000000a1 t4 : 00000014
+  [ 2334.710953]  t5 : a402303c t6 : a4023044
+  [ 2334.722077] [<c04c1660>] __do_softirq+0xb8/0x25aause: 80000005
+  [ 2334.727704] [<c000f266>] irq_exit+0x8c/0xbc
+  [ 2334.732791] [<c04bafb2>] generic_handle_arch_irq+0x64/0x74
+  [ 2334.739199] [<c0003190>] ret_from_exception+0x0/0x16
+  Enclave said: "Hello World"
+
+It's not known yet the root cause of the issue but one possible cause is some application using floating point trigger the error.
+
+
+Network Issue
+#############
+
+During startup, the DHCP client more often than not is unable to get IP address from DHCP server. You need to run the DHCP client manually after boot and it will be able to get the IP address and connect to the network
+
+.. code-block:: bash
+
+  # udhcpc
+  udhcpc: started, v1.36.0
+  udhcpc: broadcasting discover
+  udhcpc: broadcasting select for 192.168.1.4, server 192.168.1.1
+  udhcpc: lease of 192.168.1.4 obtained from 192.168.1.1, lease time 86400
+  deleting routers
+  adding dns 192.168.1.1
 
-   insmod keystone-driver.ko
-   ./tests.ke
diff --git a/examples/attestation/CMakeLists.txt b/examples/attestation/CMakeLists.txt
@@ -3,9 +3,15 @@ set(eapp_src eapp/attestor.c)
 set(host_bin attestor-runner)
 set(host_src host/attestor-runner.cpp host/host.cpp host/verifier.cpp)
 set(package_name "attestor.ke")
-set(package_script "./attestor-runner attestor eyrie-rt loader.bin --sm-bin fw_jump.bin")
 set(eyrie_plugins "none")
 
+
+if(NOT DEFINED fw_bin)
+   set(fw_bin ../../../images/fw_jump.bin)
+endif()
+get_filename_component(fw_file ${fw_bin} NAME)
+set(package_script "./attestor-runner attestor eyrie-rt loader.bin --sm-bin ${fw_file}")
+
 # eapp
 
 add_executable(${eapp_bin} ${eapp_src})
@@ -38,7 +44,7 @@ add_eyrie_runtime(${eapp_bin}-eyrie
 # add sm binary -- TODO: fix, should come from upper levels
 if(NOT DEFINED fw_bin)
   message(AUTHOR_WARNING "fw_bin should be defined by upper CMake files/ build system. Setting to default")
-  set(fw_bin ../../../images/fw_jump.bin)
+  set(fw_bin ../../../images/fw_payload.bin)
 endif()
 
 # add target for packaging (see keystone.cmake)

diff --git a/linux-keystone-driver/Makefile b/linux-keystone-driver/Makefile
@@ -14,7 +14,7 @@ PWD := $(shell pwd)
 LINUXSRC := $(PWD)/../linux
 
 default:
-	$(MAKE) -C $(LINUXSRC) ARCH=riscv CROSS_COMPILE=riscv64-unknown-linux-gnu- M=$(PWD)
+	$(MAKE) -C $(LINUXSRC) ARCH=riscv CROSS_COMPILE=riscv$(KEYSTONE_BITS)-unknown-linux-gnu- M=$(PWD)
 
 endif