🚨🔊 Add a better Data Abort Handler (#112)

* Add WIP of a better DataAbortHandler

* More WIP

* Cleanup

* refactor some things

* Refactor, clean up, and documentation

* More comments

* Update unwind.c

* Tweak printfs

* Add custom build of newlib

* Include libc/libm in template

* Exclude libc/libm from being included in cold image

* check whether pc is in the hot zone

* test unwind with hot/cold linking

* Add support for unwinding through hot code

* Clean up for PR

* Undo USE_PACKAGE:=1

* remove leftover include

* move __env_lock/__env_unlock to src/system/envlock.c

missed this in #155, but these two functions are moved to their own file for the same reason as __malloc_lock/__malloc_unlock were-- to follow the override scheme that newlib expects. found the right filename from here: https://newlib.sourceware.narkive.com/aXWplROp/queries-concerning-rtos-protection-in-newllib

* lint files

* Update .arclint whitespace

.arclint

@@ -7,7 +7,8 @@
 		},
 		"spelling": {
 			"type": "spelling",
-			"include": "(.*)"
+			"include": "(.*)",
+			"exclude": "(^(src|include)|(rtos/|display/lv_.*))"
 		},
 		"text": {
 			"type": "text",

Makefile

@@ -36,6 +36,7 @@ EXCLUDE_SRC_FROM_LIB+=$(foreach file, $(SRCDIR)/main,$(foreach cext,$(CEXTS),$(f
 # whatever files you want here. This line is configured to add all header files
 # that are in the the include directory get exported
 TEMPLATE_FILES=$(ROOT)/common.mk $(FWDIR)/v5.ld $(FWDIR)/v5-common.ld $(FWDIR)/v5-hot.ld
+TEMPLATE_FILES+=$(FWDIR)/libc.a $(FWDIR)/libm.a
 TEMPLATE_FILES+= $(INCDIR)/api.h $(INCDIR)/main.h $(INCDIR)/pros/*.* $(INCDIR)/display
 TEMPLATE_FILES+= $(SRCDIR)/main.cpp
 TEMPLATE_FILES+= $(ROOT)/template-gitignore

azure-build-libc.yaml

@@ -0,0 +1,85 @@
+###############################################################################
+#
+# This pipeline builds newlib with the same flags as the official toolchain,
+# but adds -funwind-tables so we can have proper unwind info inside libc
+#
+###############################################################################
+variables:
+  source_url: 'https://developer.arm.com/-/media/Files/downloads/gnu-rm/8-2018q4/gcc-arm-none-eabi-8-2018-q4-major-src.tar.bz2'
+  source_directory: 'gcc-arm-none-eabi-8-2018-q4-major'
+
+jobs:
+- job: BuildLibc
+  pool:
+    vmImage: 'ubuntu-16.04'
+  timeoutInMinutes: 0
+  steps:
+    - bash: |
+        sudo apt-get install software-properties-common
+        sudo dpkg --add-architecture i386
+        sudo add-apt-repository ppa:team-gcc-arm-embedded/ppa
+        sudo apt-get update
+        sudo apt-get install build-essential autoconf autogen bison dejagnu flex flip gawk git gperf gzip \
+                             nsis openssh-client p7zip-full perl python-dev libisl-dev scons tcl tofrodos \
+                             wget libncurses5-dev pv
+        sudo apt-get install gcc-arm-embedded
+      displayName: Install apt packages
+    - bash: |
+        curl -L $(source_url) -o $(source_directory).tar.bz2
+        pv -f $(source_directory).tar.bz2 | tar -xjf -
+      displayName: Download/extract arm-none-eabi-gcc source
+    - bash: |
+        sed -i '95s/TERM|\\/TERM|agent.jobstatus|\\/' ./build-common.sh
+        sed -i '302s/http:\/\/www.mr511.de\/software\//https:\/\/github.com\/gnu-mcu-eclipse\/files\/raw\/master\/libs\//' ./build-common.sh
+      displayName: Edit ./build-common.sh
+      workingDirectory: $(source_directory)
+    - bash: |
+        sed -i "294s/.*/saveenvvar CFLAGS_FOR_TARGET '-g -O2 -ffunction-sections -fdata-sections -funwind-tables'/" ./build-toolchain.sh
+        head -n 316 ./build-toolchain.sh > ./build-toolchain.sh
+      displayName: Edit ./build-toolchain.sh
+      workingDirectory: $(source_directory)
+    - bash: |
+        ./install-sources.sh --skip_steps=mingw32
+      displayName: Install sources
+      workingDirectory: $(source_directory)
+    - bash: |
+        ./build-prerequisites.sh --skip_steps=mingw32
+      displayName: Build prerequisites
+      workingDirectory: $(source_directory)
+    - bash: |
+        export CFLAGS_FOR_TARGET='-g -O2 -ffunction-sections -fdata-sections -funwind-tables'
+        export ROOT=`pwd`
+        mkdir -p $ROOT/build-native/newlib
+        pushd $ROOT/build-native/newlib
+        $ROOT/src/newlib/configure                                                                          \
+          --build="`uname -m | sed 'y/XI/xi/'`"-linux-gnu --host="`uname -m | sed 'y/XI/xi/'`"-linux-gnu    \
+          --target=arm-none-eabi                                                                            \
+          --prefix=$ROOT/newlib                                                                             \
+          --enable-newlib-io-long-long                                                                      \
+          --enable-newlib-io-c99-formats                                                                    \
+          --enable-newlib-register-fini                                                                     \
+          --enable-newlib-retargetable-locking                                                              \
+          --disable-newlib-supplied-syscalls                                                                \
+          --disable-nls                                                                                     \
+
+        make -j`grep ^processor /proc/cpuinfo|wc -l`
+        make install
+
+        ls -R $ROOT/newlib
+        ls -R $ROOT/build-native/newlib
+
+        cp $ROOT/newlib/arm-none-eabi/lib/thumb/v7-ar/libc.a $(Build.ArtifactStagingDirectory)
+        cp $ROOT/newlib/arm-none-eabi/lib/thumb/v7-ar/libm.a $(Build.ArtifactStagingDirectory)
+        ls $(Build.ArtifactStagingDirectory)
+
+        TARGET_LIBRARIES=`find $(Build.ArtifactStagingDirectory) -name \*.a`
+        for target_lib in $TARGET_LIBRARIES ; do
+          echo Stripping $target_lib
+          arm-none-eabi-objcopy -R .comment -R .note -R .debug_info -R .debug_aranges -R .debug_pubnames -R .debug_pubtypes -R .debug_abbrev -R .debug_line -R .debug_str -R .debug_ranges -R .debug_loc $target_lib || true
+        done
+      displayName: Build toolchain
+      workingDirectory: $(source_directory)
+    - task: PublishPipelineArtifact@0
+      inputs:
+        artifactName: 'newlib'
+        targetPath: $(Build.ArtifactStagingDirectory)

common.mk

@@ -1,9 +1,9 @@
 ARCHTUPLE=arm-none-eabi-
 DEVICE=VEX EDR V5
 
-MFLAGS=-mcpu=cortex-a9 -mfpu=neon-fp16 -mfloat-abi=softfp
-CPPFLAGS=-D_POSIX_THREADS -D_UNIX98_THREAD_MUTEX_ATTRIBUTES -Os
-GCCFLAGS=-ffunction-sections -fdata-sections -fdiagnostics-color
+MFLAGS=-mcpu=cortex-a9 -mfpu=neon-fp16 -mfloat-abi=softfp -Os -g
+CPPFLAGS=-D_POSIX_THREADS -D_UNIX98_THREAD_MUTEX_ATTRIBUTES
+GCCFLAGS=-ffunction-sections -fdata-sections -fdiagnostics-color -funwind-tables
 
 WARNFLAGS+=
 
@@ -12,13 +12,15 @@ SPACE +=
 COMMA := ,
 
 LIBRARIES+=$(wildcard $(FWDIR)/*.a)
-COLD_LIBRARIES= $(filter-out $(EXCLUDE_COLD_LIBRARIES), $(LIBRARIES))
+# Cannot include newlib and libc because not all of the req'd stubs are implemented
+EXCLUDE_COLD_LIBRARIES+=$(FWDIR)/libc.a $(FWDIR)/libm.a
+COLD_LIBRARIES=$(filter-out $(EXCLUDE_COLD_LIBRARIES), $(LIBRARIES))
 wlprefix=-Wl,$(subst $(SPACE),$(COMMA),$1)
 LNK_FLAGS=--gc-sections --start-group $(strip $(LIBRARIES)) -lc -lm -lgcc -lstdc++ -lsupc++ --end-group
 
 ASMFLAGS=$(MFLAGS) $(WARNFLAGS)
 CFLAGS=$(MFLAGS) $(CPPFLAGS) $(WARNFLAGS) $(GCCFLAGS) --std=gnu11
-CXXFLAGS=$(MFLAGS) $(CPPFLAGS) $(WARNFLAGS) -funwind-tables $(GCCFLAGS) --std=gnu++17
+CXXFLAGS=$(MFLAGS) $(CPPFLAGS) $(WARNFLAGS) $(GCCFLAGS) --std=gnu++17
 LDFLAGS=$(MFLAGS) $(WARNFLAGS) -nostdlib $(GCCFLAGS)
 SIZEFLAGS=-d --common
 NUMFMTFLAGS=--to=iec --format %.2f --suffix=B

firmware/v5-common.ld

@@ -1,9 +1,17 @@
 _HEAP_SIZE = DEFINED(_HEAP_SIZE) ? _HEAP_SIZE : 0x02E00000;  /* ~48 MB */
 
+start_of_cold_mem = 0x03800000;
+_COLD_MEM_SIZE = 0x04800000;
+end_of_cold_mem = start_of_cold_mem + _COLD_MEM_SIZE;
+
+start_of_hot_mem = 0x07800000;
+_HOT_MEM_SIZE = 0x00800000;
+end_of_hot_mem = start_of_hot_mem + _HOT_MEM_SIZE;
+
 MEMORY
 {
    /* user code  72M */
-   COLD_MEMORY : ORIGIN = 0x03800000, LENGTH = 0x04800000 /* Just under 19 MB */
+   COLD_MEMORY : ORIGIN = start_of_cold_mem, LENGTH = _COLD_MEM_SIZE /* Just under 19 MB */
    HEAP : ORIGIN = 0x04A00000, LENGTH = _HEAP_SIZE
-   HOT_MEMORY : ORIGIN = 0x07800000, LENGTH = 0x00800000  /* Just over 8 MB */
+   HOT_MEMORY : ORIGIN = start_of_hot_mem, LENGTH = _HOT_MEM_SIZE  /* Just over 8 MB */
 }

include/system/hot.h

@@ -4,6 +4,9 @@ struct hot_table {
 	char const* compile_timestamp;
 	char const* compile_directory;
 
+	void* __exidx_start;
+	void* __exidx_end;
+
 	struct {
 #define FUNC(F) void (*F)();
 #include "system/user_functions/list.h"

src/rtos/port.c

@@ -213,7 +213,7 @@ __attribute__(( used )) const uint32_t ulMaxAPIPriorityMask = ( configMAX_API_CA
 
 /*-----------------------------------------------------------*/
 
-static void task_clean_up() {
+void task_clean_up() {
 	// TODO: Make this a kernel debugging statement
 	// vexDisplayString(1, "Reaping dying task: %s", task_get_name(NULL));
 	task_delete(NULL);

src/system/dev/vfs.c

@@ -89,7 +89,8 @@ int vfs_update_entry(int file, struct fs_driver const* const driver, void* arg)
 	return 0;
 }
 
-int _open_r(struct _reent* r, const char* file, int flags, int mode) {
+int _open(const char* file, int flags, int mode) {
+	struct _reent* r = _REENT;
 	// Check if the filename is too long or not NULL terminated
 	size_t i = 0;
 	for (i = 0; i < MAX_FILELEN; i++) {
@@ -114,7 +115,8 @@ int _open_r(struct _reent* r, const char* file, int flags, int mode) {
 	return -1;
 }
 
-ssize_t _write_r(struct _reent* r, int file, const void* buf, size_t len) {
+ssize_t _write(int file, const void* buf, size_t len) {
+	struct _reent* r = _REENT;
 	if (file < 0 || !gid_check(&file_table_gids, file)) {
 		r->_errno = EBADF;
 		kprintf("BAD write %d", file);
@@ -123,7 +125,8 @@ ssize_t _write_r(struct _reent* r, int file, const void* buf, size_t len) {
 	return file_table[file].driver->write_r(r, file_table[file].arg, buf, len);
 }
 
-ssize_t _read_r(struct _reent* r, int file, void* buf, size_t len) {
+ssize_t _read(int file, void* buf, size_t len) {
+	struct _reent* r = _REENT;
 	if (file < 0 || !gid_check(&file_table_gids, file)) {
 		r->_errno = EBADF;
 		kprintf("BAD read %d", file);
@@ -132,7 +135,8 @@ ssize_t _read_r(struct _reent* r, int file, void* buf, size_t len) {
 	return file_table[file].driver->read_r(r, file_table[file].arg, buf, len);
 }
 
-int _close_r(struct _reent* r, int file) {
+int _close(int file) {
+	struct _reent* r = _REENT;
 	// NOTE: newlib automatically closes all open files for a given task when
 	// the task is deleted.
 	if (file > 0 && file < RESERVED_FILENOS) {
@@ -151,7 +155,8 @@ int _close_r(struct _reent* r, int file) {
 	return ret;
 }
 
-int _fstat_r(struct _reent* r, int file, struct stat* st) {
+int _fstat(int file, struct stat* st) {
+	struct _reent* r = _REENT;
 	if (file < 0 || !gid_check(&file_table_gids, file)) {
 		r->_errno = EBADF;
 		kprintf("BAD fstat %d", file);
@@ -160,7 +165,8 @@ int _fstat_r(struct _reent* r, int file, struct stat* st) {
 	return file_table[file].driver->fstat_r(r, file_table[file].arg, st);
 }
 
-off_t _lseek_r(struct _reent* r, int file, off_t ptr, int dir) {
+off_t _lseek(int file, off_t ptr, int dir) {
+	struct _reent* r = _REENT;
 	if (file < 0 || !gid_check(&file_table_gids, file)) {
 		r->_errno = EBADF;
 		kprintf("BAD lseek %d", file);
@@ -169,7 +175,8 @@ off_t _lseek_r(struct _reent* r, int file, off_t ptr, int dir) {
 	return file_table[file].driver->lseek_r(r, file_table[file].arg, ptr, dir);
 }
 
-int _isatty_r(struct _reent* r, int file) {
+int _isatty(int file) {
+	struct _reent* r = _REENT;
 	if (file < 0 || !gid_check(&file_table_gids, file)) {
 		r->_errno = EBADF;
 		kprintf("BAD isatty %d", file);

src/system/envlock.c

@@ -0,0 +1,24 @@
+/**
+ * \file system/envlock.c
+ *
+ * environment lock newlib stubs
+ *
+ * Contains implementations of environment-locking functions for newlib.
+ *
+ * Copyright (c) 2017-2019, Purdue University ACM SIGBots.
+ * All rights reserved.
+ *
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/.
+ */
+
+#include "rtos/task.h"
+
+void __env_lock(void) {
+	rtos_suspend_all();
+}
+
+void __env_unlock(void) {
+	rtos_resume_all();
+}

src/system/hot.c

@@ -1,17 +1,16 @@
-#include "kapi.h"
 #include "system/hot.h"
+#include "kapi.h"
 #include "v5_api.h"
 
-
 // stored only in cold
-struct hot_table __HOT_TABLE = { 0 };
+struct hot_table __HOT_TABLE = {0};
 struct hot_table* const HOT_TABLE = &__HOT_TABLE;
 
 #define MAGIC0 0x52616368
 #define MAGIC1 0x8CEF7310
 
-__attribute__((section (".hot_magic"))) uint32_t MAGIC[] = {MAGIC0, MAGIC1};
-uint32_t const volatile * const MAGIC_ADDR = MAGIC;
+__attribute__((section(".hot_magic"))) uint32_t MAGIC[] = {MAGIC0, MAGIC1};
+uint32_t const volatile* const MAGIC_ADDR = MAGIC;
 
 // The linker decides on these symbols in each section just as normal
 // When linking in hot, these pointers work just like any other weak symbol
@@ -20,58 +19,62 @@ uint32_t const volatile * const MAGIC_ADDR = MAGIC;
 extern char const* _PROS_COMPILE_TIMESTAMP;
 extern char const* _PROS_COMPILE_DIRECTORY;
 
+extern unsigned __exidx_start;
+extern unsigned __exidx_end;
+
 // this expands to a bunch of:
 // extern void autonomous();
 #define FUNC(F) void F();
 #include "system/user_functions/list.h"
 #undef FUNC
 
-__attribute__((section (".hot_init")))
-void install_hot_table(struct hot_table* const tbl) {
-  // printf("Hot initializing\n");
-  tbl->compile_timestamp = _PROS_COMPILE_TIMESTAMP;
-  tbl->compile_directory = _PROS_COMPILE_DIRECTORY;
+__attribute__((section(".hot_init"))) void install_hot_table(struct hot_table* const tbl) {
+	// printf("Hot initializing\n");
+	tbl->compile_timestamp = _PROS_COMPILE_TIMESTAMP;
+	tbl->compile_directory = _PROS_COMPILE_DIRECTORY;
+	tbl->__exidx_start = &__exidx_start;
+	tbl->__exidx_end = &__exidx_end;
 
-  // this expands to a bunch of:
-  // tbl->functions.autonomous = autonomous;
-  #define FUNC(F) tbl->functions.F = F;
-  #include "system/user_functions/list.h"
-  #undef FUNC
+// this expands to a bunch of:
+// tbl->functions.autonomous = autonomous;
+#define FUNC(F) tbl->functions.F = F;
+#include "system/user_functions/list.h"
+#undef FUNC
 
-  // all of these weak symbols are given to us by the linker
-  // These values should come from the hot region, since that's where this
-  // function is linked
-  extern __attribute__((weak)) uint8_t* __sbss_start[];
-  extern __attribute__((weak)) uint8_t* __sbss_end[];
-  memset(__sbss_start, 0, (size_t)__sbss_end - (size_t)__sbss_start);
+	// all of these weak symbols are given to us by the linker
+	// These values should come from the hot region, since that's where this
+	// function is linked
+	extern __attribute__((weak)) uint8_t* __sbss_start[];
+	extern __attribute__((weak)) uint8_t* __sbss_end[];
+	memset(__sbss_start, 0, (size_t)__sbss_end - (size_t)__sbss_start);
 
-  extern __attribute__((weak)) uint8_t* __bss_start[];
-  extern __attribute__((weak)) uint8_t* __bss_end[];
-  memset(__bss_start, 0, (size_t)__bss_end - (size_t)__bss_start);
+	extern __attribute__((weak)) uint8_t* __bss_start[];
+	extern __attribute__((weak)) uint8_t* __bss_end[];
+	memset(__bss_start, 0, (size_t)__bss_end - (size_t)__bss_start);
 
-  extern __attribute__((weak)) void (* const __preinit_array_start[])(void);
-  extern __attribute__((weak)) void (* const __preinit_array_end[])(void);
-  for(void (* const *ctor)() = __preinit_array_start; ctor < __preinit_array_end; ctor++) {
-    (*ctor)();
-  }
+	extern __attribute__((weak)) void (*const __preinit_array_start[])(void);
+	extern __attribute__((weak)) void (*const __preinit_array_end[])(void);
+	for (void (*const* ctor)() = __preinit_array_start; ctor < __preinit_array_end; ctor++) {
+		(*ctor)();
+	}
 
-  extern __attribute__((weak)) void (* const __init_array_start[])(void);
-  extern __attribute__((weak)) void (* const __init_array_end[])(void);
-  for(void (* const *ctor)() = __init_array_start; ctor < __init_array_end; ctor++) {
-    (*ctor)();
-  }
+	extern __attribute__((weak)) void (*const __init_array_start[])(void);
+	extern __attribute__((weak)) void (*const __init_array_end[])(void);
+	for (void (*const* ctor)() = __init_array_start; ctor < __init_array_end; ctor++) {
+		(*ctor)();
+	}
 }
 
 // this function really exists on the cold section! Called by pros_init
 // this does the check if we're running with hot/cold and invokes the hot table
 // installer (install_hot_table) located in hot memory
 void invoke_install_hot_table() {
-  // install_hot_table is at 0x07800008
-  // MAGIC_ADDR is at 0x0780000
-  // printf("%s %p %p %x %x\n", __FUNCTION__, (void*)install_hot_table, (void*)HOT_TABLE, MAGIC_ADDR[0], MAGIC_ADDR[1]);
-  if(vexSystemLinkAddrGet() == (uint32_t)0x03800000 && MAGIC_ADDR[0] == MAGIC0 && MAGIC_ADDR[1] == MAGIC1) {
-    install_hot_table(HOT_TABLE);
-  } else {
-    memset(HOT_TABLE, 0, sizeof(*HOT_TABLE));
-  }
+	// install_hot_table is at 0x07800008
+	// MAGIC_ADDR is at 0x0780000
+	// printf("%s %p %p %x %x\n", __FUNCTION__, (void*)install_hot_table, (void*)HOT_TABLE, MAGIC_ADDR[0], MAGIC_ADDR[1]);
+	if (vexSystemLinkAddrGet() == (uint32_t)0x03800000 && MAGIC_ADDR[0] == MAGIC0 && MAGIC_ADDR[1] == MAGIC1) {
+		install_hot_table(HOT_TABLE);
+	} else {
+		memset(HOT_TABLE, 0, sizeof(*HOT_TABLE));
+	}
 }