µTVM CRT modifications for on-device RPC server (apache#5921)

* Reorganize CRT into parts, public API, and add standalone build.

 * Create a make-based build in src/runtime/crt. This is intended to
   be built in build/standalone_crt (generated by running ninja
   standalone_crt in build/). Its job is to build CRT without
   depending on headers not explicitly allowed in CRT.
 * Create a "public-facing" CRT API targeted to firmware running
   alongside CRT in include/tvm/runtime/crt. Developers who are
   integrating the CRT are the target of this API.
 * Reorganize CRT internally into common/ and graph_runtime/
   pieces. Build each pieces as a separate statically-linked library.
 * Slim down TVMGraphRuntime public-facing API to just the functions
   that are used externally.
 * Updates to apps/bundle_deploy to make this work.

* Add TVMFuncRegistry, CRT test infrastructure, and tests.

 * Also add error_codes.h, a file containing error codes returned by CRT.

* Add TVMErrorf()

* [API_CHANGE] Integrate func registry into CRT.

 * NOTE: This changes the default API for functions exposed under the
   CRT by the TVMFuncCall API. `resource_handle` is now always given
   as a new 6th parameter.
 * `resource_handle` is NULL when invoked on a global function and a
   pointer to the module owning the function otherwise.

* Generalize arena-based memory manager.

* lint

* Fix git-clang-format arg parsing

* add apache header

* add mutable func registry tests

* git-clang-format

* fix more lint

* Move memory_test to crttests.

* fix tests

* checkpoint

* checkpoint

* bundle_deploy demo_static works

* rm debug printf

* git-clang-format

* fix lint

* add asf header

* pylint

* update build configs for jenkins

* make regression compiler happy

* fix build errors in regression GCC

* address comments

* git-clang-format

* fix for 32-bit cpp regression

* fix incorrect use of memcpy and tests for 32-bit

* clang-format

CMakeLists.txt

@@ -320,6 +320,7 @@ endif(USE_EXAMPLE_EXT_RUNTIME)
 
 # Module rules
 include(cmake/modules/VTA.cmake)
+include(cmake/modules/StandaloneCrt.cmake)
 include(cmake/modules/CUDA.cmake)
 include(cmake/modules/Hexagon.cmake)
 include(cmake/modules/OpenCL.cmake)

Makefile

@@ -51,6 +51,9 @@ vta:
 cpptest:
 	@mkdir -p $(OUTPUTDIR) && cd $(OUTPUTDIR) && cmake .. && $(MAKE) cpptest
 
+crttest:
+	@mkdir -p build && cd build && cmake .. && $(MAKE) crttest
+
 # EMCC; Web related scripts
 EMCC_FLAGS= -std=c++11 -DDMLC_LOG_STACK_TRACE=0\
 	-Oz -s RESERVED_FUNCTION_POINTERS=2 -s MAIN_MODULE=1 -s NO_EXIT_RUNTIME=1\

apps/bundle_deploy/Makefile

@@ -19,20 +19,25 @@
 
 # Setup build environment
 TVM_ROOT=$(shell cd ../..; pwd)
+CRT_ROOT ?= ../../src/runtime/crt
+
 DMLC_CORE=${TVM_ROOT}/3rdparty/dmlc-core
-PKG_CXXFLAGS = -Wall -std=c++14 -O2 -fPIC \
+PKG_CXXFLAGS = -g -Wall -std=c++14 -O2 -fPIC \
 	-I${TVM_ROOT}/include \
 	-I${DMLC_CORE}/include \
-	-I${TVM_ROOT}/3rdparty/dlpack/include
-PKG_CFLAGS = -Wall -std=c99 -O2 -fPIC \
+	-I${TVM_ROOT}/3rdparty/dlpack/include \
+	-Icrt_config
+PKG_CFLAGS = -g -Wall -std=c99 -O2 -fPIC \
 	-I${TVM_ROOT}/include \
 	-I${DMLC_CORE}/include \
-	-I${TVM_ROOT}/3rdparty/dlpack/include
+	-I${TVM_ROOT}/3rdparty/dlpack/include \
+	-Icrt_config
 
 PKG_LDFLAGS = -pthread
 
 build_dir := build
 
+
 demo_dynamic: $(build_dir)/demo_dynamic $(build_dir)/bundle.so $(build_dir)/bundle_c.so $(build_dir)/cat.bin
 	TVM_NUM_THREADS=1 $(build_dir)/demo_dynamic $(build_dir)/bundle.so $(build_dir)/cat.bin
 	TVM_NUM_THREADS=1 $(build_dir)/demo_dynamic $(build_dir)/bundle_c.so $(build_dir)/cat.bin
@@ -47,6 +52,12 @@ demo_static: $(build_dir)/demo_static $(build_dir)/cat.bin
 test_static: $(build_dir)/test_static $(build_dir)/test_data.bin $(build_dir)/test_output.bin
 	TVM_NUM_THREADS=1 $(build_dir)/test_static $(build_dir)/test_data.bin $(build_dir)/test_output.bin $(build_dir)/test_graph.json $(build_dir)/test_params.bin
 
+$(build_dir)/crt/graph_runtime/libgraph_runtime.a:
+	cd $(CRT_ROOT) && make QUIET= BUILD_DIR=$(abspath $(build_dir))/crt CRT_CONFIG=$(abspath crt_config/crt_config.h) graph_runtime
+
+$(build_dir)/crt/common/libcommon.a:
+	cd $(CRT_ROOT) && make QUIET= BUILD_DIR=$(abspath $(build_dir))/crt CRT_CONFIG=$(abspath crt_config/crt_config.h) common
+
 $(build_dir)/demo_dynamic: demo.cc ${build_dir}/graph.json.c ${build_dir}/params.bin.c
 	@mkdir -p $(@D)
 	g++ $(PKG_CXXFLAGS) -o $@  demo.cc -ldl
@@ -55,11 +66,14 @@ $(build_dir)/test_dynamic: test.cc ${build_dir}/test_graph.json ${build_dir}/tes
 	@mkdir -p $(@D)
 	g++ $(PKG_CXXFLAGS) -o $@  test.cc -ldl
 
-$(build_dir)/demo_static: demo_static.c ${build_dir}/bundle_static.o ${build_dir}/model.o ${build_dir}/graph.json.c ${build_dir}/params.bin.c
+$(build_dir)/model.o: $(build_dir)/model.c
+	gcc $(PKG_CFLAGS) -c -o $@ $^
+
+$(build_dir)/demo_static: demo_static.c ${build_dir}/bundle_static.o ${build_dir}/func_registry.c ${build_dir}/model.o ${build_dir}/graph.json.c ${build_dir}/params.bin.c ${build_dir}/crt/graph_runtime/libgraph_runtime.a ${build_dir}/crt/common/libcommon.a
 	@mkdir -p $(@D)
-	gcc $(PKG_CFLAGS) -o $@ demo_static.c ${build_dir}/bundle_static.o ${build_dir}/model.o -lm
+	gcc $(PKG_CFLAGS) -o $@ demo_static.c ${build_dir}/bundle_static.o ${build_dir}/func_registry.c ${build_dir}/model.o -lm ${build_dir}/crt/graph_runtime/libgraph_runtime.a ${build_dir}/crt/common/libcommon.a
 
-$(build_dir)/test_static: test_static.c ${build_dir}/bundle_static.o ${build_dir}/test_model.o
+$(build_dir)/test_static: test_static.c ${build_dir}/bundle_static.o ${build_dir}/test_func_registry.c ${build_dir}/test_model.o ${build_dir}/crt/graph_runtime/libgraph_runtime.a ${build_dir}/crt/common/libcommon.a
 	@mkdir -p $(@D)
 	gcc $(PKG_CFLAGS) -o $@ $^
 
@@ -71,27 +85,33 @@ $(build_dir)/graph.json.c: $(build_dir)/graph.json
 $(build_dir)/params.bin.c: $(build_dir)/params.bin
 	xxd -i $^  > $@
 
-$(build_dir)/model.o $(build_dir)/graph.json $(build_dir)/params.bin $(build_dir)/cat.bin: build_model.py
+$(build_dir)/func_registry.c $(build_dir)/model.c $(build_dir)/graph.json $(build_dir)/params.bin $(build_dir)/cat.bin: build_model.py
 	python3 $< -o $(build_dir)
 
-$(build_dir)/test_model.o $(build_dir)/test_graph.json $(build_dir)/test_params.bin $(build_dir)/test_data.bin $(build_dir)/test_output.bin: build_model.py
+$(build_dir)/test_func_registry.c $(build_dir)/test_model.c $(build_dir)/test_graph.json $(build_dir)/test_params.bin $(build_dir)/test_data.bin $(build_dir)/test_output.bin: build_model.py
 	python3 $< -o $(build_dir) --test
 
+$(build_dir)/test_model.o: $(build_dir)/test_model.c
+	gcc $(PKG_CFLAGS) -c -o $@ $^
+
+$(build_dir)/func_registry.o: $(build_dir)/func_registry.c
+	gcc $(PKG_CFLAGS) -c -o $@ $^
+
 # Build our bundle against the serialized bundle.c API, the runtime.cc API, and
 # the serialized graph.json and params.bin
-$(build_dir)/bundle.so: bundle.cc runtime.cc $(build_dir)/model.o
+$(build_dir)/bundle.so: bundle.cc $(build_dir)/model.o $(build_dir)/func_registry.o ${build_dir}/crt/graph_runtime/libgraph_runtime.a ${build_dir}/crt/common/libcommon.a
 	@mkdir -p $(@D)
 	g++ -shared $(PKG_CXXFLAGS) -fvisibility=hidden -o $@  $^ $(PKG_LDFLAGS)
 
-$(build_dir)/bundle_c.so: bundle.c runtime.c $(build_dir)/model.o
+$(build_dir)/bundle_c.so: bundle.c runtime.c $(build_dir)/model.o $(build_dir)/func_registry.c
 	@mkdir -p $(@D)
 	gcc -shared $(PKG_CFLAGS) -fvisibility=hidden -o $@  $^ $(PKG_LDFLAGS)
 
-$(build_dir)/test_bundle.so: bundle.cc runtime.cc $(build_dir)/test_model.o
+$(build_dir)/test_bundle.so: bundle.cc runtime.cc $(build_dir)/test_model.o $(build_dir)/test_func_registry.c
 	@mkdir -p $(@D)
 	g++ -shared $(PKG_CXXFLAGS) -fvisibility=hidden -o $@  $^ $(PKG_LDFLAGS)
 
-$(build_dir)/test_bundle_c.so: bundle.c runtime.c $(build_dir)/test_model.o
+$(build_dir)/test_bundle_c.so: bundle.c runtime.c $(build_dir)/test_model.o $(build_dir)/test_func_registry.c
 	@mkdir -p $(@D)
 	gcc -shared $(PKG_CFLAGS) -fvisibility=hidden -o $@  $^ $(PKG_LDFLAGS)
 
@@ -100,7 +120,7 @@ $(build_dir)/bundle_static.o: bundle_static.c
 	gcc -c $(PKG_CFLAGS) -o $@  $^
 
 clean:
-	rm -rf $(build_dir)/bundle.so $(build_dir)/bundle_c.so $(build_dir)/test_bundle.so $(build_dir)/test_bundle_c.so
+	rm -rf $(build_dir)/bundle.so $(build_dir)/bundle_c.so $(build_dir)/test_bundle.so $(build_dir)/test_bundle_c.so $(build_dir)/crt
 
 cleanall:
 	rm -rf $(build_dir)

apps/bundle_deploy/build_model.py

@@ -21,6 +21,7 @@
 from tvm import relay
 import tvm
 from tvm import te
+from tvm.micro import func_registry
 import logging
 import json
 
@@ -33,19 +34,21 @@ def build_module(opts):
     func = mod["main"]
     func = relay.Function(func.params, relay.nn.softmax(func.body), None, func.type_params, func.attrs)
 
-    with tvm.transform.PassContext(opt_level=3):
+    with tvm.transform.PassContext(opt_level=3, config={'tir.disable_vectorize': True}):
         graph, lib, params = relay.build(
-            func, 'llvm --system-lib', params=params)
+            func, 'c', params=params)
 
     build_dir = os.path.abspath(opts.out_dir)
     if not os.path.isdir(build_dir):
         os.makedirs(build_dir)
 
-    lib.save(os.path.join(build_dir, 'model.o'))
+    lib.save(os.path.join(build_dir, 'model.c'), 'cc')
     with open(os.path.join(build_dir, 'graph.json'), 'w') as f_graph_json:
         f_graph_json.write(graph)
     with open(os.path.join(build_dir, 'params.bin'), 'wb') as f_params:
         f_params.write(relay.save_param_dict(params))
+    func_registry.graph_json_to_c_func_registry(os.path.join(build_dir, 'graph.json'),
+                                                os.path.join(build_dir, 'func_registry.c'))
 
 def build_test_module(opts):
     import numpy as np
@@ -57,20 +60,23 @@ def build_test_module(opts):
     x_data = np.random.rand(10, 5).astype('float32')
     y_data = np.random.rand(1, 5).astype('float32')
     params = {"y": y_data}
-    graph, lib, params = relay.build(
-        tvm.IRModule.from_expr(func), "llvm --system-lib", params=params)
+    with tvm.transform.PassContext(opt_level=3, config={'tir.disable_vectorize': True}):
+        graph, lib, params = relay.build(
+            tvm.IRModule.from_expr(func), "c", params=params)
 
     build_dir = os.path.abspath(opts.out_dir)
     if not os.path.isdir(build_dir):
         os.makedirs(build_dir)
 
-    lib.save(os.path.join(build_dir, 'test_model.o'))
+    lib.save(os.path.join(build_dir, 'test_model.c'), 'cc')
     with open(os.path.join(build_dir, 'test_graph.json'), 'w') as f_graph_json:
         f_graph_json.write(graph)
     with open(os.path.join(build_dir, 'test_params.bin'), 'wb') as f_params:
         f_params.write(relay.save_param_dict(params))
     with open(os.path.join(build_dir, "test_data.bin"), "wb") as fp:
         fp.write(x_data.astype(np.float32).tobytes())
+    func_registry.graph_json_to_c_func_registry(os.path.join(build_dir, 'test_graph.json'),
+                                                os.path.join(build_dir, 'test_func_registry.c'))
     x_output = x_data + y_data
     with open(os.path.join(build_dir, "test_output.bin"), "wb") as fp:
         fp.write(x_output.astype(np.float32).tobytes())

apps/bundle_deploy/bundle.c

@@ -49,6 +49,8 @@ TVM_DLL void* tvm_runtime_create(const char* json_data, const char* params_data,
   TVMModuleHandle (*TVMGraphRuntimeCreate)(const char*, const TVMModuleHandle, const TVMContext*);
   int (*TVMGraphRuntime_LoadParams)(TVMModuleHandle, const char*, const uint32_t);
 
+  TVM_CCALL(TVMRuntimeInitialize());
+
   // get pointers
   TVM_CCALL(TVMFuncGetGlobal("runtime.SystemLib", (TVMFunctionHandle*)&SystemLibraryCreate));
   TVM_CCALL(

apps/bundle_deploy/bundle_static.c

@@ -19,9 +19,21 @@
 
 #include <stdio.h>
 #include <stdlib.h>
+#include <tvm/runtime/crt/crt.h>
+#include <tvm/runtime/crt/graph_runtime.h>
+#include <tvm/runtime/crt/packed_func.h>
 
 #include "bundle.h"
-#include "runtime.c"
+
+/*! \brief macro to do C API call */
+#define TVM_CCALL(func)                                                              \
+  do {                                                                               \
+    tvm_crt_error_t ret = (func);                                                    \
+    if (ret != kTvmErrorNoError) {                                                   \
+      fprintf(stderr, "%s: %d: error: %s\n", __FILE__, __LINE__, TVMGetLastError()); \
+      exit(ret);                                                                     \
+    }                                                                                \
+  } while (0)
 
 TVM_DLL void* tvm_runtime_create(const char* json_data, const char* params_data,
                                  const uint64_t params_size) {
@@ -36,44 +48,43 @@ TVM_DLL void* tvm_runtime_create(const char* json_data, const char* params_data,
   ctx.device_type = (DLDeviceType)device_type;
   ctx.device_id = device_id;
 
-  // declare pointers
-  void* (*SystemLibraryCreate)();
-  TVMGraphRuntime* (*TVMGraphRuntimeCreate)(const char*, const TVMModuleHandle, const TVMContext*);
-  int (*TVMGraphRuntime_LoadParams)(TVMModuleHandle, const char*, const uint32_t);
-
   // get pointers
-  TVMFuncGetGlobal("runtime.SystemLib", (TVMFunctionHandle*)&SystemLibraryCreate);
-  TVMFuncGetGlobal("tvm.graph_runtime.create", (TVMFunctionHandle*)&TVMGraphRuntimeCreate);
+  TVM_CCALL(TVMInitializeRuntime());
+  TVMPackedFunc pf;
+  TVMArgs args = TVMArgs_Create(NULL, NULL, 0);
+  TVM_CCALL(TVMPackedFunc_InitGlobalFunc(&pf, "runtime.SystemLib", &args));
+  TVM_CCALL(TVMPackedFunc_Call(&pf));
+
+  TVMModuleHandle mod_syslib = TVMArgs_AsModuleHandle(&pf.ret_value, 0);
 
   // run modules
-  TVMModuleHandle mod_syslib = SystemLibraryCreate();
-  TVMModuleHandle mod = TVMGraphRuntimeCreate(json_data, mod_syslib, &ctx);
-  TVMModGetFunction(mod, "load_params", 0, (TVMFunctionHandle*)&TVMGraphRuntime_LoadParams);
-  TVMGraphRuntime_LoadParams(mod, params.data, params.size);
+  TVMGraphRuntime* graph_runtime = TVMGraphRuntime_Create(json_data, mod_syslib, &ctx);
+  TVMGraphRuntime_LoadParams(graph_runtime, params.data, params.size);
 
-  return mod;
+  return graph_runtime;
 }
 
 TVM_DLL void tvm_runtime_destroy(void* runtime) {
-  void (*TVMGraphRuntimeRelease)(TVMModuleHandle*);
-  TVMFuncGetGlobal("tvm.graph_runtime.release", (TVMFunctionHandle*)&TVMGraphRuntimeRelease);
-  TVMGraphRuntimeRelease(&runtime);
+  TVMGraphRuntime* graph_runtime = (TVMGraphRuntime*)runtime;
+  TVMGraphRuntime_Release(&graph_runtime);
 }
 
 TVM_DLL void tvm_runtime_set_input(void* runtime, const char* name, DLTensor* tensor) {
-  void (*TVMGraphRuntime_SetInput)(TVMModuleHandle, const char*, DLTensor*);
-  TVMFuncGetGlobal("tvm.graph_runtime.set_input", (TVMFunctionHandle*)&TVMGraphRuntime_SetInput);
-  TVMGraphRuntime_SetInput(runtime, name, tensor);
+  TVMGraphRuntime* graph_runtime = (TVMGraphRuntime*)runtime;
+  TVMGraphRuntime_SetInput(graph_runtime, name, tensor);
 }
 
 TVM_DLL void tvm_runtime_run(void* runtime) {
-  void (*TVMGraphRuntime_Run)(TVMModuleHandle runtime);
-  TVMFuncGetGlobal("tvm.graph_runtime.run", (TVMFunctionHandle*)&TVMGraphRuntime_Run);
-  TVMGraphRuntime_Run(runtime);
+  TVMGraphRuntime* graph_runtime = (TVMGraphRuntime*)runtime;
+  TVMGraphRuntime_Run(graph_runtime);
 }
 
 TVM_DLL void tvm_runtime_get_output(void* runtime, int32_t index, DLTensor* tensor) {
-  int (*TVMGraphRuntime_GetOutput)(TVMModuleHandle, const int32_t, DLTensor*);
-  TVMFuncGetGlobal("tvm.graph_runtime.get_output", (TVMFunctionHandle*)&TVMGraphRuntime_GetOutput);
-  TVMGraphRuntime_GetOutput(runtime, index, tensor);
-}
+  TVMGraphRuntime* graph_runtime = (TVMGraphRuntime*)runtime;
+  TVMGraphRuntime_GetOutput(graph_runtime, index, tensor);
+}
+
+void __attribute__((noreturn)) TVMPlatformAbort(int error_code) {
+  fprintf(stderr, "TVMPlatformAbort: %d\n", error_code);
+  exit(-1);
+}

apps/bundle_deploy/runtime.c → apps/bundle_deploy/crt_config/crt_config.h

@@ -17,11 +17,12 @@
  * under the License.
  */
 
-/* Explicitly declare posix_memalign function */
-#if _POSIX_C_SOURCE < 200112L
-#undef _POSIX_C_SOURCE
-#define _POSIX_C_SOURCE 200809L
-#endif
+/*!
+ * \file apps/bundle_deploy/crt_config.h
+ * \brief CRT configuration for bundle_deploy app.
+ */
+#ifndef TVM_RUNTIME_CRT_CONFIG_H_
+#define TVM_RUNTIME_CRT_CONFIG_H_
 
 /*! Support low-level debugging in MISRA-C runtime */
 #define TVM_CRT_DEBUG 0
@@ -56,11 +57,12 @@
 #define TVM_CRT_LOG_VIRT_MEM_SIZE 24
 
 /*! \brief Page size for virtual memory allocation */
-#define TVM_CRT_PAGE_BYTES 4096
+#define TVM_CRT_PAGE_BYTES_LOG 12
+
+/*! Maximum number of registered modules. */
+#define TVM_CRT_MAX_REGISTERED_MODULES 2
+
+/*! Size of the global function registry, in bytes. */
+#define TVM_CRT_GLOBAL_FUNC_REGISTRY_SIZE_BYTES 200
 
-#include "../../src/runtime/crt/crt_backend_api.c"
-#include "../../src/runtime/crt/crt_runtime_api.c"
-#include "../../src/runtime/crt/graph_runtime.c"
-#include "../../src/runtime/crt/load_json.c"
-#include "../../src/runtime/crt/memory.c"
-#include "../../src/runtime/crt/ndarray.c"
+#endif  // TVM_RUNTIME_CRT_CONFIG_H_

cmake/config.cmake

@@ -226,3 +226,6 @@ set(USE_FALLBACK_STL_MAP OFF)
 # Whether to use hexagon device
 set(USE_HEXAGON_DEVICE OFF)
 set(USE_HEXAGON_SDK /path/to/sdk)
+
+# Whether to compile the standalone C runtime.
+set(USE_STANDALONE_CRT ON)