µTVM CRT modifications for on-device RPC server

CMakeLists.txt

@@ -305,6 +305,7 @@ endif(USE_EXAMPLE_EXT_RUNTIME)
 
 # Module rules
 include(cmake/modules/VTA.cmake)
+include(cmake/modules/StandaloneCrt.cmake)
 include(cmake/modules/CUDA.cmake)
 include(cmake/modules/Hexagon.cmake)
 include(cmake/modules/OpenCL.cmake)

Makefile

@@ -49,6 +49,9 @@ vta:
 cpptest:
 	@mkdir -p build && cd build && cmake .. && $(MAKE) cpptest
 
+crttest:
+	@mkdir -p build && cd build && cmake .. && $(MAKE) crttest
+
 # EMCC; Web related scripts
 EMCC_FLAGS= -std=c++11 -DDMLC_LOG_STACK_TRACE=0\
 	-Oz -s RESERVED_FUNCTION_POINTERS=2 -s MAIN_MODULE=1 -s NO_EXIT_RUNTIME=1\

apps/bundle_deploy/Makefile

@@ -19,20 +19,25 @@
 
 # Setup build environment
 TVM_ROOT=$(shell cd ../..; pwd)
+CRT_ROOT ?= ../../src/runtime/crt
+
 DMLC_CORE=${TVM_ROOT}/3rdparty/dmlc-core
 PKG_CXXFLAGS = -Wall -std=c++14 -O2 -fPIC \
 	-I${TVM_ROOT}/include \
 	-I${DMLC_CORE}/include \
-	-I${TVM_ROOT}/3rdparty/dlpack/include
+	-I${TVM_ROOT}/3rdparty/dlpack/include \
+	-Icrt_config
 PKG_CFLAGS = -Wall -std=c99 -O2 -fPIC \
 	-I${TVM_ROOT}/include \
 	-I${DMLC_CORE}/include \
-	-I${TVM_ROOT}/3rdparty/dlpack/include
+	-I${TVM_ROOT}/3rdparty/dlpack/include \
+	-Icrt_config
 
 PKG_LDFLAGS = -pthread
 
 build_dir := build
 
+
 demo_dynamic: $(build_dir)/demo_dynamic $(build_dir)/bundle.so $(build_dir)/bundle_c.so $(build_dir)/cat.bin
 	TVM_NUM_THREADS=1 $(build_dir)/demo_dynamic $(build_dir)/bundle.so $(build_dir)/cat.bin
 	TVM_NUM_THREADS=1 $(build_dir)/demo_dynamic $(build_dir)/bundle_c.so $(build_dir)/cat.bin
@@ -47,6 +52,12 @@ demo_static: $(build_dir)/demo_static $(build_dir)/cat.bin
 test_static: $(build_dir)/test_static $(build_dir)/test_data.bin $(build_dir)/test_output.bin
 	TVM_NUM_THREADS=1 $(build_dir)/test_static $(build_dir)/test_data.bin $(build_dir)/test_output.bin $(build_dir)/test_graph.json $(build_dir)/test_params.bin
 
+$(build_dir)/crt/graph_runtime/libgraph_runtime.a:
+	cd $(CRT_ROOT) && make QUIET= BUILD_DIR=$(abspath $(build_dir))/crt CRT_CONFIG=$(abspath crt_config/crt_config.h) graph_runtime
+
+$(build_dir)/crt/common/libcommon.a:
+	cd $(CRT_ROOT) && make QUIET= BUILD_DIR=$(abspath $(build_dir))/crt CRT_CONFIG=$(abspath crt_config/crt_config.h) common
+
 $(build_dir)/demo_dynamic: demo.cc ${build_dir}/graph.json.c ${build_dir}/params.bin.c
 	@mkdir -p $(@D)
 	g++ $(PKG_CXXFLAGS) -o $@  demo.cc -ldl
@@ -55,11 +66,11 @@ $(build_dir)/test_dynamic: test.cc ${build_dir}/test_graph.json ${build_dir}/tes
 	@mkdir -p $(@D)
 	g++ $(PKG_CXXFLAGS) -o $@  test.cc -ldl
 
-$(build_dir)/demo_static: demo_static.c ${build_dir}/bundle_static.o ${build_dir}/model.o ${build_dir}/graph.json.c ${build_dir}/params.bin.c
+$(build_dir)/demo_static: demo_static.c ${build_dir}/bundle_static.o ${build_dir}/model.o ${build_dir}/graph.json.c ${build_dir}/params.bin.c ${build_dir}/crt/graph_runtime/libgraph_runtime.a ${build_dir}/crt/common/libcommon.a
 	@mkdir -p $(@D)
-	gcc $(PKG_CFLAGS) -o $@ demo_static.c ${build_dir}/bundle_static.o ${build_dir}/model.o -lm
+	gcc $(PKG_CFLAGS) -o $@ demo_static.c ${build_dir}/bundle_static.o ${build_dir}/model.o -lm ${build_dir}/crt/graph_runtime/libgraph_runtime.a ${build_dir}/crt/common/libcommon.a
 
-$(build_dir)/test_static: test_static.c ${build_dir}/bundle_static.o ${build_dir}/test_model.o
+$(build_dir)/test_static: test_static.c ${build_dir}/bundle_static.o ${build_dir}/test_model.o ${build_dir}/crt/graph_runtime/libgraph_runtime.a ${build_dir}/crt/common/libcommon.a
 	@mkdir -p $(@D)
 	gcc $(PKG_CFLAGS) -o $@ $^
 
@@ -79,7 +90,7 @@ $(build_dir)/test_model.o $(build_dir)/test_graph.json $(build_dir)/test_params.
 
 # Build our bundle against the serialized bundle.c API, the runtime.cc API, and
 # the serialized graph.json and params.bin
-$(build_dir)/bundle.so: bundle.cc runtime.cc $(build_dir)/model.o
+$(build_dir)/bundle.so: bundle.cc $(build_dir)/model.o ${build_dir}/crt/graph_runtime/libgraph_runtime.a ${build_dir}/crt/common/libcommon.a
 	@mkdir -p $(@D)
 	g++ -shared $(PKG_CXXFLAGS) -fvisibility=hidden -o $@  $^ $(PKG_LDFLAGS)
 
@@ -100,7 +111,7 @@ $(build_dir)/bundle_static.o: bundle_static.c
 	gcc -c $(PKG_CFLAGS) -o $@  $^
 
 clean:
-	rm -rf $(build_dir)/bundle.so $(build_dir)/bundle_c.so $(build_dir)/test_bundle.so $(build_dir)/test_bundle_c.so
+	rm -rf $(build_dir)/bundle.so $(build_dir)/bundle_c.so $(build_dir)/test_bundle.so $(build_dir)/test_bundle_c.so $(build_dir)/crt
 
 cleanall:
 	rm -rf $(build_dir)

apps/bundle_deploy/bundle_static.c

@@ -19,9 +19,9 @@
 
 #include <stdio.h>
 #include <stdlib.h>
+#include <tvm/runtime/crt/graph_runtime.h>
 
 #include "bundle.h"
-#include "runtime.c"
 
 TVM_DLL void* tvm_runtime_create(const char* json_data, const char* params_data,
                                  const uint64_t params_size) {
@@ -38,7 +38,8 @@ TVM_DLL void* tvm_runtime_create(const char* json_data, const char* params_data,
 
   // declare pointers
   void* (*SystemLibraryCreate)();
-  TVMGraphRuntime* (*TVMGraphRuntimeCreate)(const char*, const TVMModuleHandle, const TVMContext*);
+  TVMGraphRuntimeAPI* (*TVMGraphRuntimeCreate)(const char*, const TVMModuleHandle,
+                                               const TVMContext*);
   int (*TVMGraphRuntime_LoadParams)(TVMModuleHandle, const char*, const uint32_t);
 
   // get pointers
@@ -76,4 +77,4 @@ TVM_DLL void tvm_runtime_get_output(void* runtime, int32_t index, DLTensor* tens
   int (*TVMGraphRuntime_GetOutput)(TVMModuleHandle, const int32_t, DLTensor*);
   TVMFuncGetGlobal("tvm.graph_runtime.get_output", (TVMFunctionHandle*)&TVMGraphRuntime_GetOutput);
   TVMGraphRuntime_GetOutput(runtime, index, tensor);
-}
+}

apps/bundle_deploy/runtime.c → apps/bundle_deploy/crt_config/crt_config.h

@@ -17,11 +17,12 @@
  * under the License.
  */
 
-/* Explicitly declare posix_memalign function */
-#if _POSIX_C_SOURCE < 200112L
-#undef _POSIX_C_SOURCE
-#define _POSIX_C_SOURCE 200809L
-#endif
+/*!
+ * \file apps/bundle_deploy/crt_config.h
+ * \brief CRT configuration for bundle_deploy app.
+ */
+#ifndef TVM_RUNTIME_CRT_CONFIG_H_
+#define TVM_RUNTIME_CRT_CONFIG_H_
 
 /*! Support low-level debugging in MISRA-C runtime */
 #define TVM_CRT_DEBUG 0
@@ -58,9 +59,4 @@
 /*! \brief Page size for virtual memory allocation */
 #define TVM_CRT_PAGE_BYTES 4096
 
-#include "../../src/runtime/crt/crt_backend_api.c"
-#include "../../src/runtime/crt/crt_runtime_api.c"
-#include "../../src/runtime/crt/graph_runtime.c"
-#include "../../src/runtime/crt/load_json.c"
-#include "../../src/runtime/crt/memory.c"
-#include "../../src/runtime/crt/ndarray.c"
+#endif  // TVM_RUNTIME_CRT_CONFIG_H_

cmake/modules/StandaloneCrt.cmake

@@ -0,0 +1,134 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+if(USE_STANDALONE_CRT)
+  message(STATUS "Build with standalone CRT")
+  file(GLOB crt_srcs src/runtime/crt/**)
+
+  function(tvm_crt_add_copy_file var src dest)
+    get_filename_component(basename "${src}" NAME)
+    get_filename_component(dest_parent_dir "${dest}" DIRECTORY)
+    add_custom_command(
+        OUTPUT "${dest}"
+        COMMAND "${CMAKE_COMMAND}" -E copy "${src}" "${dest}"
+        DEPENDS "${src}")
+    list(APPEND "${var}" "${dest}")
+    set("${var}" "${${var}}" PARENT_SCOPE)
+  endfunction(tvm_crt_add_copy_file)
+
+  # Build an isolated build directory, separate from the TVM tree.
+  file(GLOB_RECURSE crt_srcs
+       RELATIVE "${CMAKE_SOURCE_DIR}/src/runtime/crt"
+       "${CMAKE_SOURCE_DIR}/src/runtime/crt/common/*.c"
+       "${CMAKE_SOURCE_DIR}/src/runtime/crt/graph_runtime/*.c"
+       "${CMAKE_SOURCE_DIR}/src/runtime/crt/include/*.h")
+
+  foreach(src IN LISTS crt_srcs)
+    tvm_crt_add_copy_file(host_isolated_build_deps ${CMAKE_SOURCE_DIR}/src/runtime/crt/${src} standalone_crt/${src})
+  endforeach()
+
+  file(GLOB_RECURSE crt_headers RELATIVE "${CMAKE_SOURCE_DIR}/include" include/tvm/runtime/crt/*.h)
+  foreach(hdr IN LISTS crt_headers)
+    tvm_crt_add_copy_file(host_isolated_build_deps ${CMAKE_SOURCE_DIR}/include/${hdr} standalone_crt/include/${hdr})
+  endforeach()
+
+  tvm_crt_add_copy_file(host_isolated_build_deps
+      ${CMAKE_SOURCE_DIR}/include/tvm/runtime/c_runtime_api.h standalone_crt/include/tvm/runtime/c_runtime_api.h)
+  tvm_crt_add_copy_file(host_isolated_build_deps
+      ${CMAKE_SOURCE_DIR}/include/tvm/runtime/c_backend_api.h standalone_crt/include/tvm/runtime/c_backend_api.h)
+  tvm_crt_add_copy_file(host_isolated_build_deps
+      ${CMAKE_SOURCE_DIR}/src/runtime/crt/Makefile standalone_crt/Makefile)
+
+  get_filename_component(crt_config_abspath src/runtime/crt/host/crt_config.h ABSOLUTE)
+  list(APPEND host_isolated_build_deps src/runtime/crt/host/crt_config.h)
+
+  get_filename_component(host_build_dir_abspath "${CMAKE_CURRENT_BINARY_DIR}/host_standalone_crt" ABSOLUTE)
+
+  if(${VERBOSE})
+  set(make_quiet QUIET=)
+  else(${VERBOSE})
+  set(make_quiet )
+  endif(${VERBOSE})
+  add_custom_command(
+      OUTPUT host_standalone_crt/common/libcommon.a host_standalone_crt/graph_runtime/libgraph_runtime.a
+      COMMAND make
+          DLPACK_INCLUDE_DIR=${CMAKE_SOURCE_DIR}/3rdparty/dlpack/include
+          TVM_INCLUDE_DIR=${CMAKE_CURRENT_BINARY_DIR}/standalone_crt/include
+          CRT_CONFIG=${crt_config_abspath}
+          BUILD_DIR=${host_build_dir_abspath} all ${make_quiet}
+      WORKING_DIRECTORY standalone_crt
+      DEPENDS ${host_isolated_build_deps})
+  add_custom_target(host_standalone_crt SOURCES  host_standalone_crt/common/libcommon.a host_standalone_crt/graph_runtime/libgraph_runtime.a)
+
+#  add_custom_target(host_standalone_crt ALL
+#      DEPENDS host_standalone_crt/common/libcommon.a host_standalone_crt/graph_runtime/libgraph_runtime.a)
+  add_library(host_standalone_crt_common INTERFACE)
+  add_dependencies(host_standalone_crt_common host_standalone_crt)
+  target_link_libraries(host_standalone_crt_common
+      INTERFACE ${CMAKE_CURRENT_BINARY_DIR}/crt/common/libcommon.a)
+  # set_target_properties(host_standalone_crt_common PROPERTIES
+  #     IMPORTED_LOCATION ${CMAKE_CURRENT_BINARY_DIR}/crt/common
+  #     IMPORTED_OBJECTS ${CMAKE_CURRENT_BINARY_DIR}/crt/common/libcommon.a
+  #     PUBLIC_HEADER ${crt_headers})
+  add_dependencies(host_standalone_crt_common host_standalone_crt)
+#      ${CMAKE_CURRENT_BINARY_DIR}/host_standalone_crt/common/libcommon.a)
+
+  add_library(host_standalone_crt_graph_runtime INTERFACE IMPORTED)
+  add_dependencies(host_standalone_crt_graph_runtime host_standalone_crt)
+  target_link_libraries(host_standalone_crt_graph_runtime
+      INTERFACE ${CMAKE_CURRENT_BINARY_DIR}/crt/graph_runtime/libgraph_runtime.a)
+
+  # set_target_properties(host_standalone_crt_graph_runtime PROPERTIES
+  #     IMPORTED_LOCATION ${CMAKE_CURRENT_BINARY_DIR}/crt/graph_runtime
+  #     IMPORTED_OBJECTS ${CMAKE_CURRENT_BINARY_DIR}/crt/graph_runtime/libgraph_runtime.a
+  #     PUBLIC_HEADER ${crt_headers})
+  add_dependencies(host_standalone_crt_graph_runtime host_standalone_crt)
+#      ${CMAKE_CURRENT_BINARY_DIR}/host_standalone_crt/graph_runtime/libgraph_runtime.a)
+
+  # Standalone CRT tests
+  file(GLOB TEST_SRCS ${CMAKE_SOURCE_DIR}/tests/crt/*.cc)
+  find_path(GTEST_INCLUDE_DIR gtest/gtest.h)
+  find_library(GTEST_LIB gtest "$ENV{GTEST_LIB}")
+
+  # Create the `crttest` target if we can find GTest.  If not, we create dummy
+  # targets that give the user an informative error message.
+  if(GTEST_INCLUDE_DIR AND GTEST_LIB)
+    foreach(__srcpath ${TEST_SRCS})
+      get_filename_component(__srcname ${__srcpath} NAME)
+      string(REPLACE ".cc" "" __execname ${__srcname})
+      add_executable(${__execname} ${__srcpath})
+      list(APPEND TEST_EXECS ${__execname})
+      target_include_directories(${__execname} PUBLIC ${GTEST_INCLUDE_DIR})
+      target_link_directories(${__execname} PRIVATE
+          ${CMAKE_CURRENT_BINARY_DIR}/host_standalone_crt/common
+          ${CMAKE_CURRENT_BINARY_DIR}/host_standalone_crt/graph_runtime)
+      target_link_libraries(${__execname} common graph_runtime ${GTEST_LIB})
+      set_target_properties(${__execname} PROPERTIES EXCLUDE_FROM_ALL 1)
+      set_target_properties(${__execname} PROPERTIES EXCLUDE_FROM_DEFAULT_BUILD 1)
+    endforeach()
+    add_custom_target(crttest DEPENDS ${TEST_EXECS})
+  elseif(NOT GTEST_INCLUDE_DIR)
+    add_custom_target(crttest
+        COMMAND echo "Missing Google Test headers in include path"
+        COMMAND exit 1)
+  elseif(NOT GTEST_LIB)
+    add_custom_target(crttest
+        COMMAND echo "Missing Google Test library"
+        COMMAND exit 1)
+  endif()
+
+endif(USE_STANDALONE_CRT)

include/tvm/runtime/c_backend_api.h

@@ -42,11 +42,13 @@ extern "C" {
  * \param num_args Number of arguments.
  * \param out_ret_value The output value of the the return value.
  * \param out_ret_tcode The output type code of the return value.
+ * \param resource_handle Pointer to associated resource.
  *
  * \return 0 if success, -1 if failure happens, set error via TVMAPISetLastError.
  */
 typedef int (*TVMBackendPackedCFunc)(TVMValue* args, int* type_codes, int num_args,
-                                     TVMValue* out_ret_value, int* out_ret_tcode);
+                                     TVMValue* out_ret_value, int* out_ret_tcode,
+                                     void* resource_handle);
 
 /*!
  * \brief Backend function for modules to get function

include/tvm/runtime/crt/error_codes.h

@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/*!
+ * \file src/runtime/crt/error_codes.h
+ * \brief Defines integral error codes returned by the CRT.
+ */
+#ifndef TVM_RUNTIME_CRT_ERROR_CODES_H_
+#define TVM_RUNTIME_CRT_ERROR_CODES_H_
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#define TVM_CRT_ERROR_CATEGORY_Pos 8
+#define TVM_CRT_ERROR_CATEGORY_Msk (0xff << TVM_CRT_ERROR_CATEGORY_Pos)
+#define TVM_CRT_ERROR_CODE_Pos 0
+#define TVM_CRT_ERROR_CODE_Msk (0xff << TVM_CRT_ERROR_CODE_Pos)
+
+#define DEFINE_TVM_CRT_ERROR(category, code) \
+  (((category) << TVM_CRT_ERROR_CATEGORY_Pos) | ((code) << TVM_CRT_ERROR_CODE_Pos))
+typedef enum { kTvmErrorCategoryFunctionRegistry = 1 } tvm_crt_error_category_t;
+
+typedef enum {
+  kTvmErrorNoError = 0,
+
+  // Function Registry
+  kTvmErrorFunctionNameNotFound = DEFINE_TVM_CRT_ERROR(kTvmErrorCategoryFunctionRegistry, 0),
+  kTvmErrorFunctionIndexInvalid = DEFINE_TVM_CRT_ERROR(kTvmErrorCategoryFunctionRegistry, 1),
+  kTvmErrorFunctionRegistryFull = DEFINE_TVM_CRT_ERROR(kTvmErrorCategoryFunctionRegistry, 2),
+  kTvmErrorFunctionAlreadyDefined = DEFINE_TVM_CRT_ERROR(kTvmErrorCategoryFunctionRegistry, 3),
+} tvm_crt_error_t;
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif  // TVM_RUNTIME_CRT_ERROR_CODES_H_