diff --git a/CMakeLists.txt b/CMakeLists.txt index 967c06e27264..d18b2a3b03a1 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -320,6 +320,7 @@ endif(USE_EXAMPLE_EXT_RUNTIME) # Module rules include(cmake/modules/VTA.cmake) +include(cmake/modules/StandaloneCrt.cmake) include(cmake/modules/CUDA.cmake) include(cmake/modules/Hexagon.cmake) include(cmake/modules/OpenCL.cmake) diff --git a/Makefile b/Makefile index 9139be664494..9063cd1b8fff 100644 --- a/Makefile +++ b/Makefile @@ -51,6 +51,9 @@ vta: cpptest: @mkdir -p $(OUTPUTDIR) && cd $(OUTPUTDIR) && cmake .. && $(MAKE) cpptest +crttest: + @mkdir -p build && cd build && cmake .. && $(MAKE) crttest + # EMCC; Web related scripts EMCC_FLAGS= -std=c++11 -DDMLC_LOG_STACK_TRACE=0\ -Oz -s RESERVED_FUNCTION_POINTERS=2 -s MAIN_MODULE=1 -s NO_EXIT_RUNTIME=1\ diff --git a/apps/bundle_deploy/Makefile b/apps/bundle_deploy/Makefile index 73f9d75b12f8..eeea539d16d2 100644 --- a/apps/bundle_deploy/Makefile +++ b/apps/bundle_deploy/Makefile @@ -19,20 +19,25 @@ # Setup build environment TVM_ROOT=$(shell cd ../..; pwd) +CRT_ROOT ?= ../../src/runtime/crt + DMLC_CORE=${TVM_ROOT}/3rdparty/dmlc-core -PKG_CXXFLAGS = -Wall -std=c++14 -O2 -fPIC \ +PKG_CXXFLAGS = -g -Wall -std=c++14 -O2 -fPIC \ -I${TVM_ROOT}/include \ -I${DMLC_CORE}/include \ - -I${TVM_ROOT}/3rdparty/dlpack/include -PKG_CFLAGS = -Wall -std=c99 -O2 -fPIC \ + -I${TVM_ROOT}/3rdparty/dlpack/include \ + -Icrt_config +PKG_CFLAGS = -g -Wall -std=c99 -O2 -fPIC \ -I${TVM_ROOT}/include \ -I${DMLC_CORE}/include \ - -I${TVM_ROOT}/3rdparty/dlpack/include + -I${TVM_ROOT}/3rdparty/dlpack/include \ + -Icrt_config PKG_LDFLAGS = -pthread build_dir := build + demo_dynamic: $(build_dir)/demo_dynamic $(build_dir)/bundle.so $(build_dir)/bundle_c.so $(build_dir)/cat.bin TVM_NUM_THREADS=1 $(build_dir)/demo_dynamic $(build_dir)/bundle.so $(build_dir)/cat.bin TVM_NUM_THREADS=1 $(build_dir)/demo_dynamic $(build_dir)/bundle_c.so $(build_dir)/cat.bin @@ -47,6 +52,12 @@ demo_static: $(build_dir)/demo_static $(build_dir)/cat.bin test_static: $(build_dir)/test_static $(build_dir)/test_data.bin $(build_dir)/test_output.bin TVM_NUM_THREADS=1 $(build_dir)/test_static $(build_dir)/test_data.bin $(build_dir)/test_output.bin $(build_dir)/test_graph.json $(build_dir)/test_params.bin +$(build_dir)/crt/graph_runtime/libgraph_runtime.a: + cd $(CRT_ROOT) && make QUIET= BUILD_DIR=$(abspath $(build_dir))/crt CRT_CONFIG=$(abspath crt_config/crt_config.h) graph_runtime + +$(build_dir)/crt/common/libcommon.a: + cd $(CRT_ROOT) && make QUIET= BUILD_DIR=$(abspath $(build_dir))/crt CRT_CONFIG=$(abspath crt_config/crt_config.h) common + $(build_dir)/demo_dynamic: demo.cc ${build_dir}/graph.json.c ${build_dir}/params.bin.c @mkdir -p $(@D) g++ $(PKG_CXXFLAGS) -o $@ demo.cc -ldl @@ -55,11 +66,14 @@ $(build_dir)/test_dynamic: test.cc ${build_dir}/test_graph.json ${build_dir}/tes @mkdir -p $(@D) g++ $(PKG_CXXFLAGS) -o $@ test.cc -ldl -$(build_dir)/demo_static: demo_static.c ${build_dir}/bundle_static.o ${build_dir}/model.o ${build_dir}/graph.json.c ${build_dir}/params.bin.c +$(build_dir)/model.o: $(build_dir)/model.c + gcc $(PKG_CFLAGS) -c -o $@ $^ + +$(build_dir)/demo_static: demo_static.c ${build_dir}/bundle_static.o ${build_dir}/func_registry.c ${build_dir}/model.o ${build_dir}/graph.json.c ${build_dir}/params.bin.c ${build_dir}/crt/graph_runtime/libgraph_runtime.a ${build_dir}/crt/common/libcommon.a @mkdir -p $(@D) - gcc $(PKG_CFLAGS) -o $@ demo_static.c ${build_dir}/bundle_static.o ${build_dir}/model.o -lm + gcc $(PKG_CFLAGS) -o $@ demo_static.c ${build_dir}/bundle_static.o ${build_dir}/func_registry.c ${build_dir}/model.o -lm ${build_dir}/crt/graph_runtime/libgraph_runtime.a ${build_dir}/crt/common/libcommon.a -$(build_dir)/test_static: test_static.c ${build_dir}/bundle_static.o ${build_dir}/test_model.o +$(build_dir)/test_static: test_static.c ${build_dir}/bundle_static.o ${build_dir}/test_func_registry.c ${build_dir}/test_model.o ${build_dir}/crt/graph_runtime/libgraph_runtime.a ${build_dir}/crt/common/libcommon.a @mkdir -p $(@D) gcc $(PKG_CFLAGS) -o $@ $^ @@ -71,27 +85,33 @@ $(build_dir)/graph.json.c: $(build_dir)/graph.json $(build_dir)/params.bin.c: $(build_dir)/params.bin xxd -i $^ > $@ -$(build_dir)/model.o $(build_dir)/graph.json $(build_dir)/params.bin $(build_dir)/cat.bin: build_model.py +$(build_dir)/func_registry.c $(build_dir)/model.c $(build_dir)/graph.json $(build_dir)/params.bin $(build_dir)/cat.bin: build_model.py python3 $< -o $(build_dir) -$(build_dir)/test_model.o $(build_dir)/test_graph.json $(build_dir)/test_params.bin $(build_dir)/test_data.bin $(build_dir)/test_output.bin: build_model.py +$(build_dir)/test_func_registry.c $(build_dir)/test_model.c $(build_dir)/test_graph.json $(build_dir)/test_params.bin $(build_dir)/test_data.bin $(build_dir)/test_output.bin: build_model.py python3 $< -o $(build_dir) --test +$(build_dir)/test_model.o: $(build_dir)/test_model.c + gcc $(PKG_CFLAGS) -c -o $@ $^ + +$(build_dir)/func_registry.o: $(build_dir)/func_registry.c + gcc $(PKG_CFLAGS) -c -o $@ $^ + # Build our bundle against the serialized bundle.c API, the runtime.cc API, and # the serialized graph.json and params.bin -$(build_dir)/bundle.so: bundle.cc runtime.cc $(build_dir)/model.o +$(build_dir)/bundle.so: bundle.cc $(build_dir)/model.o $(build_dir)/func_registry.o ${build_dir}/crt/graph_runtime/libgraph_runtime.a ${build_dir}/crt/common/libcommon.a @mkdir -p $(@D) g++ -shared $(PKG_CXXFLAGS) -fvisibility=hidden -o $@ $^ $(PKG_LDFLAGS) -$(build_dir)/bundle_c.so: bundle.c runtime.c $(build_dir)/model.o +$(build_dir)/bundle_c.so: bundle.c runtime.c $(build_dir)/model.o $(build_dir)/func_registry.c @mkdir -p $(@D) gcc -shared $(PKG_CFLAGS) -fvisibility=hidden -o $@ $^ $(PKG_LDFLAGS) -$(build_dir)/test_bundle.so: bundle.cc runtime.cc $(build_dir)/test_model.o +$(build_dir)/test_bundle.so: bundle.cc runtime.cc $(build_dir)/test_model.o $(build_dir)/test_func_registry.c @mkdir -p $(@D) g++ -shared $(PKG_CXXFLAGS) -fvisibility=hidden -o $@ $^ $(PKG_LDFLAGS) -$(build_dir)/test_bundle_c.so: bundle.c runtime.c $(build_dir)/test_model.o +$(build_dir)/test_bundle_c.so: bundle.c runtime.c $(build_dir)/test_model.o $(build_dir)/test_func_registry.c @mkdir -p $(@D) gcc -shared $(PKG_CFLAGS) -fvisibility=hidden -o $@ $^ $(PKG_LDFLAGS) @@ -100,7 +120,7 @@ $(build_dir)/bundle_static.o: bundle_static.c gcc -c $(PKG_CFLAGS) -o $@ $^ clean: - rm -rf $(build_dir)/bundle.so $(build_dir)/bundle_c.so $(build_dir)/test_bundle.so $(build_dir)/test_bundle_c.so + rm -rf $(build_dir)/bundle.so $(build_dir)/bundle_c.so $(build_dir)/test_bundle.so $(build_dir)/test_bundle_c.so $(build_dir)/crt cleanall: rm -rf $(build_dir) diff --git a/apps/bundle_deploy/build_model.py b/apps/bundle_deploy/build_model.py index 1d415cd40ef4..2fe8ef34bfc5 100644 --- a/apps/bundle_deploy/build_model.py +++ b/apps/bundle_deploy/build_model.py @@ -21,6 +21,7 @@ from tvm import relay import tvm from tvm import te +from tvm.micro import func_registry import logging import json @@ -33,19 +34,21 @@ def build_module(opts): func = mod["main"] func = relay.Function(func.params, relay.nn.softmax(func.body), None, func.type_params, func.attrs) - with tvm.transform.PassContext(opt_level=3): + with tvm.transform.PassContext(opt_level=3, config={'tir.disable_vectorize': True}): graph, lib, params = relay.build( - func, 'llvm --system-lib', params=params) + func, 'c', params=params) build_dir = os.path.abspath(opts.out_dir) if not os.path.isdir(build_dir): os.makedirs(build_dir) - lib.save(os.path.join(build_dir, 'model.o')) + lib.save(os.path.join(build_dir, 'model.c'), 'cc') with open(os.path.join(build_dir, 'graph.json'), 'w') as f_graph_json: f_graph_json.write(graph) with open(os.path.join(build_dir, 'params.bin'), 'wb') as f_params: f_params.write(relay.save_param_dict(params)) + func_registry.graph_json_to_c_func_registry(os.path.join(build_dir, 'graph.json'), + os.path.join(build_dir, 'func_registry.c')) def build_test_module(opts): import numpy as np @@ -57,20 +60,23 @@ def build_test_module(opts): x_data = np.random.rand(10, 5).astype('float32') y_data = np.random.rand(1, 5).astype('float32') params = {"y": y_data} - graph, lib, params = relay.build( - tvm.IRModule.from_expr(func), "llvm --system-lib", params=params) + with tvm.transform.PassContext(opt_level=3, config={'tir.disable_vectorize': True}): + graph, lib, params = relay.build( + tvm.IRModule.from_expr(func), "c", params=params) build_dir = os.path.abspath(opts.out_dir) if not os.path.isdir(build_dir): os.makedirs(build_dir) - lib.save(os.path.join(build_dir, 'test_model.o')) + lib.save(os.path.join(build_dir, 'test_model.c'), 'cc') with open(os.path.join(build_dir, 'test_graph.json'), 'w') as f_graph_json: f_graph_json.write(graph) with open(os.path.join(build_dir, 'test_params.bin'), 'wb') as f_params: f_params.write(relay.save_param_dict(params)) with open(os.path.join(build_dir, "test_data.bin"), "wb") as fp: fp.write(x_data.astype(np.float32).tobytes()) + func_registry.graph_json_to_c_func_registry(os.path.join(build_dir, 'test_graph.json'), + os.path.join(build_dir, 'test_func_registry.c')) x_output = x_data + y_data with open(os.path.join(build_dir, "test_output.bin"), "wb") as fp: fp.write(x_output.astype(np.float32).tobytes()) diff --git a/apps/bundle_deploy/bundle.c b/apps/bundle_deploy/bundle.c index 4def96eb12b9..d86c79e0c1bc 100644 --- a/apps/bundle_deploy/bundle.c +++ b/apps/bundle_deploy/bundle.c @@ -49,6 +49,8 @@ TVM_DLL void* tvm_runtime_create(const char* json_data, const char* params_data, TVMModuleHandle (*TVMGraphRuntimeCreate)(const char*, const TVMModuleHandle, const TVMContext*); int (*TVMGraphRuntime_LoadParams)(TVMModuleHandle, const char*, const uint32_t); + TVM_CCALL(TVMRuntimeInitialize()); + // get pointers TVM_CCALL(TVMFuncGetGlobal("runtime.SystemLib", (TVMFunctionHandle*)&SystemLibraryCreate)); TVM_CCALL( diff --git a/apps/bundle_deploy/bundle_static.c b/apps/bundle_deploy/bundle_static.c index 5ecc5e58eea7..b999a7b63872 100644 --- a/apps/bundle_deploy/bundle_static.c +++ b/apps/bundle_deploy/bundle_static.c @@ -19,9 +19,21 @@ #include #include +#include +#include +#include #include "bundle.h" -#include "runtime.c" + +/*! \brief macro to do C API call */ +#define TVM_CCALL(func) \ + do { \ + tvm_crt_error_t ret = (func); \ + if (ret != kTvmErrorNoError) { \ + fprintf(stderr, "%s: %d: error: %s\n", __FILE__, __LINE__, TVMGetLastError()); \ + exit(ret); \ + } \ + } while (0) TVM_DLL void* tvm_runtime_create(const char* json_data, const char* params_data, const uint64_t params_size) { @@ -36,44 +48,43 @@ TVM_DLL void* tvm_runtime_create(const char* json_data, const char* params_data, ctx.device_type = (DLDeviceType)device_type; ctx.device_id = device_id; - // declare pointers - void* (*SystemLibraryCreate)(); - TVMGraphRuntime* (*TVMGraphRuntimeCreate)(const char*, const TVMModuleHandle, const TVMContext*); - int (*TVMGraphRuntime_LoadParams)(TVMModuleHandle, const char*, const uint32_t); - // get pointers - TVMFuncGetGlobal("runtime.SystemLib", (TVMFunctionHandle*)&SystemLibraryCreate); - TVMFuncGetGlobal("tvm.graph_runtime.create", (TVMFunctionHandle*)&TVMGraphRuntimeCreate); + TVM_CCALL(TVMInitializeRuntime()); + TVMPackedFunc pf; + TVMArgs args = TVMArgs_Create(NULL, NULL, 0); + TVM_CCALL(TVMPackedFunc_InitGlobalFunc(&pf, "runtime.SystemLib", &args)); + TVM_CCALL(TVMPackedFunc_Call(&pf)); + + TVMModuleHandle mod_syslib = TVMArgs_AsModuleHandle(&pf.ret_value, 0); // run modules - TVMModuleHandle mod_syslib = SystemLibraryCreate(); - TVMModuleHandle mod = TVMGraphRuntimeCreate(json_data, mod_syslib, &ctx); - TVMModGetFunction(mod, "load_params", 0, (TVMFunctionHandle*)&TVMGraphRuntime_LoadParams); - TVMGraphRuntime_LoadParams(mod, params.data, params.size); + TVMGraphRuntime* graph_runtime = TVMGraphRuntime_Create(json_data, mod_syslib, &ctx); + TVMGraphRuntime_LoadParams(graph_runtime, params.data, params.size); - return mod; + return graph_runtime; } TVM_DLL void tvm_runtime_destroy(void* runtime) { - void (*TVMGraphRuntimeRelease)(TVMModuleHandle*); - TVMFuncGetGlobal("tvm.graph_runtime.release", (TVMFunctionHandle*)&TVMGraphRuntimeRelease); - TVMGraphRuntimeRelease(&runtime); + TVMGraphRuntime* graph_runtime = (TVMGraphRuntime*)runtime; + TVMGraphRuntime_Release(&graph_runtime); } TVM_DLL void tvm_runtime_set_input(void* runtime, const char* name, DLTensor* tensor) { - void (*TVMGraphRuntime_SetInput)(TVMModuleHandle, const char*, DLTensor*); - TVMFuncGetGlobal("tvm.graph_runtime.set_input", (TVMFunctionHandle*)&TVMGraphRuntime_SetInput); - TVMGraphRuntime_SetInput(runtime, name, tensor); + TVMGraphRuntime* graph_runtime = (TVMGraphRuntime*)runtime; + TVMGraphRuntime_SetInput(graph_runtime, name, tensor); } TVM_DLL void tvm_runtime_run(void* runtime) { - void (*TVMGraphRuntime_Run)(TVMModuleHandle runtime); - TVMFuncGetGlobal("tvm.graph_runtime.run", (TVMFunctionHandle*)&TVMGraphRuntime_Run); - TVMGraphRuntime_Run(runtime); + TVMGraphRuntime* graph_runtime = (TVMGraphRuntime*)runtime; + TVMGraphRuntime_Run(graph_runtime); } TVM_DLL void tvm_runtime_get_output(void* runtime, int32_t index, DLTensor* tensor) { - int (*TVMGraphRuntime_GetOutput)(TVMModuleHandle, const int32_t, DLTensor*); - TVMFuncGetGlobal("tvm.graph_runtime.get_output", (TVMFunctionHandle*)&TVMGraphRuntime_GetOutput); - TVMGraphRuntime_GetOutput(runtime, index, tensor); -} \ No newline at end of file + TVMGraphRuntime* graph_runtime = (TVMGraphRuntime*)runtime; + TVMGraphRuntime_GetOutput(graph_runtime, index, tensor); +} + +void __attribute__((noreturn)) TVMPlatformAbort(int error_code) { + fprintf(stderr, "TVMPlatformAbort: %d\n", error_code); + exit(-1); +} diff --git a/apps/bundle_deploy/runtime.c b/apps/bundle_deploy/crt_config/crt_config.h similarity index 80% rename from apps/bundle_deploy/runtime.c rename to apps/bundle_deploy/crt_config/crt_config.h index 248a295f97b8..ac06ecf41ca5 100644 --- a/apps/bundle_deploy/runtime.c +++ b/apps/bundle_deploy/crt_config/crt_config.h @@ -17,11 +17,12 @@ * under the License. */ -/* Explicitly declare posix_memalign function */ -#if _POSIX_C_SOURCE < 200112L -#undef _POSIX_C_SOURCE -#define _POSIX_C_SOURCE 200809L -#endif +/*! + * \file apps/bundle_deploy/crt_config.h + * \brief CRT configuration for bundle_deploy app. + */ +#ifndef TVM_RUNTIME_CRT_CONFIG_H_ +#define TVM_RUNTIME_CRT_CONFIG_H_ /*! Support low-level debugging in MISRA-C runtime */ #define TVM_CRT_DEBUG 0 @@ -56,11 +57,12 @@ #define TVM_CRT_LOG_VIRT_MEM_SIZE 24 /*! \brief Page size for virtual memory allocation */ -#define TVM_CRT_PAGE_BYTES 4096 +#define TVM_CRT_PAGE_BYTES_LOG 12 + +/*! Maximum number of registered modules. */ +#define TVM_CRT_MAX_REGISTERED_MODULES 2 + +/*! Size of the global function registry, in bytes. */ +#define TVM_CRT_GLOBAL_FUNC_REGISTRY_SIZE_BYTES 200 -#include "../../src/runtime/crt/crt_backend_api.c" -#include "../../src/runtime/crt/crt_runtime_api.c" -#include "../../src/runtime/crt/graph_runtime.c" -#include "../../src/runtime/crt/load_json.c" -#include "../../src/runtime/crt/memory.c" -#include "../../src/runtime/crt/ndarray.c" +#endif // TVM_RUNTIME_CRT_CONFIG_H_ diff --git a/cmake/config.cmake b/cmake/config.cmake index ccdce65af881..8ca4db859860 100644 --- a/cmake/config.cmake +++ b/cmake/config.cmake @@ -226,3 +226,6 @@ set(USE_FALLBACK_STL_MAP OFF) # Whether to use hexagon device set(USE_HEXAGON_DEVICE OFF) set(USE_HEXAGON_SDK /path/to/sdk) + +# Whether to compile the standalone C runtime. +set(USE_STANDALONE_CRT ON) diff --git a/cmake/modules/StandaloneCrt.cmake b/cmake/modules/StandaloneCrt.cmake new file mode 100644 index 000000000000..8783cd757fe1 --- /dev/null +++ b/cmake/modules/StandaloneCrt.cmake @@ -0,0 +1,151 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +if(USE_STANDALONE_CRT) + include(ExternalProject) + + message(STATUS "Build with standalone CRT") + file(GLOB crt_srcs src/runtime/crt/**) + + function(tvm_crt_add_copy_file var src dest) + get_filename_component(basename "${src}" NAME) + get_filename_component(dest_parent_dir "${dest}" DIRECTORY) + add_custom_command( + OUTPUT "${dest}" + COMMAND "${CMAKE_COMMAND}" -E copy "${src}" "${dest}" + DEPENDS "${src}") + list(APPEND "${var}" "${dest}") + set("${var}" "${${var}}" PARENT_SCOPE) + endfunction(tvm_crt_add_copy_file) + + # Build an isolated build directory, separate from the TVM tree. + file(GLOB_RECURSE crt_srcs + RELATIVE "${CMAKE_SOURCE_DIR}/src/runtime/crt" + "${CMAKE_SOURCE_DIR}/src/runtime/crt/common/*.c" + "${CMAKE_SOURCE_DIR}/src/runtime/crt/graph_runtime/*.c" + "${CMAKE_SOURCE_DIR}/src/runtime/crt/include/*.h") + + foreach(src IN LISTS crt_srcs) + tvm_crt_add_copy_file(host_isolated_build_deps ${CMAKE_SOURCE_DIR}/src/runtime/crt/${src} standalone_crt/${src}) + endforeach() + + file(GLOB_RECURSE crt_headers RELATIVE "${CMAKE_SOURCE_DIR}/include" include/tvm/runtime/crt/*.h) + foreach(hdr IN LISTS crt_headers) + tvm_crt_add_copy_file(host_isolated_build_deps ${CMAKE_SOURCE_DIR}/include/${hdr} standalone_crt/include/${hdr}) + endforeach() + + tvm_crt_add_copy_file(host_isolated_build_deps + ${CMAKE_SOURCE_DIR}/include/tvm/runtime/c_runtime_api.h standalone_crt/include/tvm/runtime/c_runtime_api.h) + tvm_crt_add_copy_file(host_isolated_build_deps + ${CMAKE_SOURCE_DIR}/include/tvm/runtime/c_backend_api.h standalone_crt/include/tvm/runtime/c_backend_api.h) + tvm_crt_add_copy_file(host_isolated_build_deps + ${CMAKE_SOURCE_DIR}/src/runtime/crt/Makefile standalone_crt/Makefile) + + get_filename_component(crt_config_abspath src/runtime/crt/host/crt_config.h ABSOLUTE) + list(APPEND host_isolated_build_deps src/runtime/crt/host/crt_config.h) + add_custom_target(standalone_crt DEPENDS ${host_isolated_build_deps}) + + get_filename_component(host_build_dir_abspath "${CMAKE_CURRENT_BINARY_DIR}/host_standalone_crt" ABSOLUTE) + + if(${VERBOSE}) + set(make_quiet QUIET=) + else(${VERBOSE}) + set(make_quiet ) + endif(${VERBOSE}) + + ExternalProject_Add(host_standalone_crt + DOWNLOAD_COMMAND "" + SOURCE_DIR standalone_crt + CONFIGURE_COMMAND "" + BUILD_COMMAND make + DLPACK_INCLUDE_DIR=${CMAKE_SOURCE_DIR}/3rdparty/dlpack/include + TVM_INCLUDE_DIR=${CMAKE_CURRENT_BINARY_DIR}/standalone_crt/include + CRT_CONFIG=${crt_config_abspath} + BUILD_DIR=${host_build_dir_abspath} all ${make_quiet} + BUILD_IN_SOURCE ON + WORKING_DIRECTORY standalone_crt + COMMENT "Building host CRT runtime" + BUILD_BYPRODUCTS host_standalone_crt/common/libcommon.a host_standalone_crt/graph_runtime/libgraph_runtime.a + DEPENDS standalone_crt + INSTALL_COMMAND "" + ) + ExternalProject_Add_StepDependencies(host_standalone_crt build ${host_isolated_build_deps}) +# add_custom_command( +# OUTPUT host_standalone_crt/common/libcommon.a host_standalone_crt/graph_runtime/libgraph_runtime.a +# COMMAND make +# DLPACK_INCLUDE_DIR=${CMAKE_SOURCE_DIR}/3rdparty/dlpack/include +# TVM_INCLUDE_DIR=${CMAKE_CURRENT_BINARY_DIR}/standalone_crt/include +# CRT_CONFIG=${crt_config_abspath} +# BUILD_DIR=${host_build_dir_abspath} all ${make_quiet} +# WORKING_DIRECTORY standalone_crt +# DEPENDS ${host_isolated_build_deps}) +# add_custom_target(host_standalone_crt DEPENDS host_standalone_crt/common/libcommon.a host_standalone_crt/graph_runtime/libgraph_runtime.a) + +# # add_custom_target(host_standalone_crt ALL +# # DEPENDS host_standalone_crt/common/libcommon.a host_standalone_crt/graph_runtime/libgraph_runtime.a) + add_library(host_standalone_crt_common STATIC IMPORTED GLOBAL) + add_dependencies(host_standalone_crt_common host_standalone_crt) + set_target_properties(host_standalone_crt_common PROPERTIES + IMPORTED_LOCATION "${CMAKE_CURRENT_BINARY_DIR}/host_standalone_crt/common/libcommon.a" + IMPORTED_OBJECTS "${CMAKE_CURRENT_BINARY_DIR}/host_standalone_crt/common/libcommon.a" + PUBLIC_HEADER "${crt_headers}") +# add_dependencies(host_standalone_crt_common host_standalone_crt) +# # ${CMAKE_CURRENT_BINARY_DIR}/host_standalone_crt/common/libcommon.a) + + add_library(host_standalone_crt_graph_runtime STATIC IMPORTED GLOBAL) + add_dependencies(host_standalone_crt_graph_runtime host_standalone_crt) + set_target_properties(host_standalone_crt_graph_runtime PROPERTIES + IMPORTED_LOCATION "${CMAKE_CURRENT_BINARY_DIR}/host_standalone_crt/graph_runtime/libgraph_runtime.a" + IMPORTED_OBJECTS "${CMAKE_CURRENT_BINARY_DIR}/host_standalone_crt/graph_runtime/libgraph_runtime.a" + PUBLIC_HEADER "${crt_headers}") +# add_dependencies(host_standalone_crt_graph_runtime host_standalone_crt) +# # ${CMAKE_CURRENT_BINARY_DIR}/host_standalone_crt/graph_runtime/libgraph_runtime.a) + + # Standalone CRT tests + file(GLOB TEST_SRCS ${CMAKE_SOURCE_DIR}/tests/crt/*.cc) + find_path(GTEST_INCLUDE_DIR gtest/gtest.h) + find_library(GTEST_LIB gtest "$ENV{GTEST_LIB}") + + # Create the `crttest` target if we can find GTest. If not, we create dummy + # targets that give the user an informative error message. + if(GTEST_INCLUDE_DIR AND GTEST_LIB) + foreach(__srcpath ${TEST_SRCS}) + get_filename_component(__srcname ${__srcpath} NAME) + string(REPLACE ".cc" "" __execname ${__srcname}) + add_executable(${__execname} ${__srcpath}) + list(APPEND TEST_EXECS ${__execname}) + target_include_directories(${__execname} PUBLIC ${GTEST_INCLUDE_DIR} ${CMAKE_CURRENT_BINARY_DIR}/standalone_crt/include ${CMAKE_SOURCE_DIR}/src/runtime/crt/host) + target_compile_options(${__execname} PRIVATE -pthread) +# target_link_directories(${__execname} PRIVATE +# ${CMAKE_CURRENT_BINARY_DIR}/host_standalone_crt/common +# ${CMAKE_CURRENT_BINARY_DIR}/host_standalone_crt/graph_runtime) + target_link_libraries(${__execname} host_standalone_crt_graph_runtime host_standalone_crt_common ${GTEST_LIB} pthread) + set_target_properties(${__execname} PROPERTIES EXCLUDE_FROM_ALL 1) + set_target_properties(${__execname} PROPERTIES EXCLUDE_FROM_DEFAULT_BUILD 1) + endforeach() + add_custom_target(crttest DEPENDS ${TEST_EXECS}) + elseif(NOT GTEST_INCLUDE_DIR) + add_custom_target(crttest + COMMAND echo "Missing Google Test headers in include path" + COMMAND exit 1) + elseif(NOT GTEST_LIB) + add_custom_target(crttest + COMMAND echo "Missing Google Test library" + COMMAND exit 1) + endif() + +endif(USE_STANDALONE_CRT) diff --git a/include/tvm/runtime/c_backend_api.h b/include/tvm/runtime/c_backend_api.h index 40cef83ee05b..f74b2d37ff97 100644 --- a/include/tvm/runtime/c_backend_api.h +++ b/include/tvm/runtime/c_backend_api.h @@ -42,11 +42,13 @@ extern "C" { * \param num_args Number of arguments. * \param out_ret_value The output value of the the return value. * \param out_ret_tcode The output type code of the return value. + * \param resource_handle Pointer to associated resource. * * \return 0 if success, -1 if failure happens, set error via TVMAPISetLastError. */ typedef int (*TVMBackendPackedCFunc)(TVMValue* args, int* type_codes, int num_args, - TVMValue* out_ret_value, int* out_ret_tcode); + TVMValue* out_ret_value, int* out_ret_tcode, + void* resource_handle); /*! * \brief Backend function for modules to get function diff --git a/apps/bundle_deploy/runtime.cc b/include/tvm/runtime/crt/crt.h similarity index 53% rename from apps/bundle_deploy/runtime.cc rename to include/tvm/runtime/crt/crt.h index 8e294a05775d..c2e2af4ca5de 100644 --- a/apps/bundle_deploy/runtime.cc +++ b/include/tvm/runtime/crt/crt.h @@ -17,21 +17,29 @@ * under the License. */ -#include -#include -#include -#include +/*! + * \file tvm/runtime/crt/crt.h + * \brief Defines core life cycle functions used by CRT. + */ + +#ifndef TVM_RUNTIME_CRT_CRT_H_ +#define TVM_RUNTIME_CRT_CRT_H_ + +#include + +#ifdef __cplusplus +extern "C" { +#endif + +/*! + * \brief Initialize various data structures used by the rutnime. + * \return An error code describing the outcome of intialization. Generally, initialization + * is only expected to fail due to a misconfiguration. + */ +tvm_crt_error_t TVMInitializeRuntime(void); + +#ifdef __cplusplus +} // extern "C" +#endif -#include "../../src/runtime/c_runtime_api.cc" -#include "../../src/runtime/cpu_device_api.cc" -#include "../../src/runtime/file_util.cc" -#include "../../src/runtime/graph/graph_runtime.cc" -#include "../../src/runtime/library_module.cc" -#include "../../src/runtime/module.cc" -#include "../../src/runtime/ndarray.cc" -#include "../../src/runtime/object.cc" -#include "../../src/runtime/registry.cc" -#include "../../src/runtime/system_library.cc" -#include "../../src/runtime/thread_pool.cc" -#include "../../src/runtime/threading_backend.cc" -#include "../../src/runtime/workspace_pool.cc" +#endif // TVM_RUNTIME_CRT_CRT_H_ diff --git a/include/tvm/runtime/crt/error_codes.h b/include/tvm/runtime/crt/error_codes.h new file mode 100644 index 000000000000..aae4550a5792 --- /dev/null +++ b/include/tvm/runtime/crt/error_codes.h @@ -0,0 +1,55 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +/*! + * \file include/tvm/runtime/crt/error_codes.h + * \brief Defines integral error codes returned by the CRT. + */ +#ifndef TVM_RUNTIME_CRT_ERROR_CODES_H_ +#define TVM_RUNTIME_CRT_ERROR_CODES_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +#define TVM_CRT_ERROR_CATEGORY_Pos 8 +#define TVM_CRT_ERROR_CATEGORY_Msk (0xff << TVM_CRT_ERROR_CATEGORY_Pos) +#define TVM_CRT_ERROR_CODE_Pos 0 +#define TVM_CRT_ERROR_CODE_Msk (0xff << TVM_CRT_ERROR_CODE_Pos) + +#define DEFINE_TVM_CRT_ERROR(category, code) \ + (((category) << TVM_CRT_ERROR_CATEGORY_Pos) | ((code) << TVM_CRT_ERROR_CODE_Pos)) +typedef enum { kTvmErrorCategoryFunctionRegistry = 1 } tvm_crt_error_category_t; + +typedef enum { + kTvmErrorNoError = 0, + + // Function Registry + kTvmErrorFunctionNameNotFound = DEFINE_TVM_CRT_ERROR(kTvmErrorCategoryFunctionRegistry, 0), + kTvmErrorFunctionIndexInvalid = DEFINE_TVM_CRT_ERROR(kTvmErrorCategoryFunctionRegistry, 1), + kTvmErrorFunctionRegistryFull = DEFINE_TVM_CRT_ERROR(kTvmErrorCategoryFunctionRegistry, 2), + kTvmErrorFunctionAlreadyDefined = DEFINE_TVM_CRT_ERROR(kTvmErrorCategoryFunctionRegistry, 3), + kTvmErrorBufferTooSmall = DEFINE_TVM_CRT_ERROR(kTvmErrorCategoryFunctionRegistry, 4), +} tvm_crt_error_t; + +#ifdef __cplusplus +} +#endif + +#endif // TVM_RUNTIME_CRT_ERROR_CODES_H_ diff --git a/include/tvm/runtime/crt/func_registry.h b/include/tvm/runtime/crt/func_registry.h new file mode 100644 index 000000000000..4f8a19af591e --- /dev/null +++ b/include/tvm/runtime/crt/func_registry.h @@ -0,0 +1,137 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +/*! + * \file include/tvm/runtime/crt/func_registry.h + * \brief Defines generic string-based function lookup structs + */ +#ifndef TVM_RUNTIME_CRT_FUNC_REGISTRY_H_ +#define TVM_RUNTIME_CRT_FUNC_REGISTRY_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +#include +#include + +typedef uint16_t tvm_function_index_t; + +typedef uint16_t tvm_module_index_t; + +/*! + * \brief A data structure that facilitates function lookup by C-string name. + */ +typedef struct TVMFuncRegistry { + /*! \brief Names of registered functions, concatenated together and separated by \0. + * An additional \0 is present at the end of the concatenated blob to mark the end. + * + * Byte 0 is the number of functions in `funcs`. + */ + const char* names; + + /*! \brief Function pointers, in the same order as their names in `names`. */ + const TVMBackendPackedCFunc* funcs; +} TVMFuncRegistry; + +/*! + * \brief Get packed function from registry by name. + * + * \param reg TVMFunctionRegistry instance that contains the function. +, * \param name The function name + * \param function_index Pointer to receive the 0-based index of the function in the registry, if it + * was found. Unmodified otherwise. + * \return kTvmErrorNoError when successful. kTvmErrorFunctionNameNotFound when no function matched +`name`. + */ +tvm_crt_error_t TVMFuncRegistry_Lookup(const TVMFuncRegistry* reg, const char* name, + tvm_function_index_t* function_index); + +/*! + * \brief Fetch TVMBackendPackedCFunc given a function index + * + * \param reg TVMFunctionRegistry instance that contains the function. + * \param index Index of the function. + * \param out_func Pointer which receives the function pointer at `index`, if a valid + * index was given. Unmodified otherwise. + * \return kTvmErrorNoError when successful. kTvmErrorFunctionIndexInvalid when index was out of + * range. + */ +tvm_crt_error_t TVMFuncRegistry_GetByIndex(const TVMFuncRegistry* reg, tvm_function_index_t index, + TVMBackendPackedCFunc* out_func); + +/*! + * \brief A TVMFuncRegistry that supports adding and changing the functions. + */ +typedef struct TVMMutableFuncRegistry { + TVMFuncRegistry registry; + + /*! \brief maximum number of functions in this registry. */ + size_t max_functions; +} TVMMutableFuncRegistry; + +// Defined to work around compiler limitations. +#define TVM_AVERAGE_FUNCTION_NAME_STRLEN_BYTES 10 + +/*! + * \brief Size of an average function name in a TVMMutableFuncRegistry, in bytes. + * + * This is just an assumption made by the runtime for ease of use. + */ +static const size_t kTvmAverageFunctionNameStrlenBytes = TVM_AVERAGE_FUNCTION_NAME_STRLEN_BYTES; + +/*! + * \brief Size of an average entry in a TVMMutableFuncRegistry, in bytes. + * + * Assumes a constant average function name length. + */ +static const size_t kTvmAverageFuncEntrySizeBytes = + TVM_AVERAGE_FUNCTION_NAME_STRLEN_BYTES + 1 + sizeof(void*); + +/*! + * \brief Create a new mutable function registry from a block of memory. + * + * \param reg TVMMutableFuncRegistry to create. + * \param buffer Backing memory available for this function registry. + * \param buffer_size_bytes Number of bytes available in buffer. + * \return kTvmErrorNoError when successful. kTvmErrorBufferTooSmall when buffer_size_bytes is so + * small that a single function cannot be registered. + */ +tvm_crt_error_t TVMMutableFuncRegistry_Create(TVMMutableFuncRegistry* reg, uint8_t* buffer, + size_t buffer_size_bytes); + +/*! + * \brief Add or set a function in the registry. + * + * \param reg The mutable function registry to affect. + * \param name Name of the function. + * \param func The function pointer. + * \param override non-zero if an existing entry should be overridden. + * \return kTvmErrorNoError when successful. kTvmErrorRegistryFull when `reg` already contains + * `max_functions` entries. kTvmErrorFunctionAlreadyDefined when a function named `name` is + * already present in the registry, and `override` == 0. + */ +tvm_crt_error_t TVMMutableFuncRegistry_Set(TVMMutableFuncRegistry* reg, const char* name, + TVMBackendPackedCFunc func, int override); + +#ifdef __cplusplus +} +#endif + +#endif // TVM_RUNTIME_CRT_FUNC_REGISTRY_H_ diff --git a/include/tvm/runtime/crt/graph_runtime.h b/include/tvm/runtime/crt/graph_runtime.h new file mode 100644 index 000000000000..d2eb3b7785e9 --- /dev/null +++ b/include/tvm/runtime/crt/graph_runtime.h @@ -0,0 +1,115 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +/*! + * \file graph_runtime.h + * \brief Tiny graph runtime that can run graph containing only tvm PackedFunc. + */ +#ifndef TVM_RUNTIME_CRT_GRAPH_RUNTIME_H_ +#define TVM_RUNTIME_CRT_GRAPH_RUNTIME_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +#include +#include +#include + +struct TVMModule; + +/*! \brief operator attributes about tvm op */ +typedef struct TVMOpParam { + char func_name[120]; + uint32_t num_inputs; + uint32_t num_outputs; + uint32_t flatten_data; +} TVMOpParam; + +// Graph attribute +typedef struct TVMGraphRuntimeGraphAttr { + uint32_t storage_num_not_alloctaed; + uint32_t* storage_id; + uint32_t* device_index; + char* dltype; // "int8", "int16", "float32" + uint32_t dltype_count; + int64_t* shape; + uint32_t* ndim; + uint32_t shape_count; +} TVMGraphRuntimeGraphAttr; + +typedef struct TVMGraphRuntime TVMGraphRuntime; + +// public functions +/*! + * \brief Allocate a new GraphRuntime with vmalloc and initialize it. + * + * \param sym_json JSON-encoded graph. + * \param m TVM Module that exposes the functions to call. + * \param ctxs runtime execution context. + */ +TVMGraphRuntime* TVMGraphRuntime_Create(const char* sym_json, const struct TVMModule* m, + const TVMContext* ctxs); + +int TVMGraphRuntime_GetInputIndex(TVMGraphRuntime* runtime, const char* name); + +/*! + * \brief set input to the graph based on name. + * \param runtime The graph runtime. + * \param name The name of the input. + * \param data_in The input data. + */ +void TVMGraphRuntime_SetInput(TVMGraphRuntime* runtime, const char* name, DLTensor* data_in); + +/*! + * \brief Return NDArray for given output index. + * \param runtime The graph runtime. + * \param index The output index. + * \param out The DLTensor corresponding to given output node index. + * \return The result of this function execution. + */ +int TVMGraphRuntime_GetOutput(TVMGraphRuntime* runtime, const int32_t index, DLTensor* out); + +/*! + * \brief Load parameters from parameter blob. + * \param runtime The graph runtime. + * \param param_blob A binary blob of parameter. + * \param param_size The parameter size. + * \return The result of this function execution. + */ +int TVMGraphRuntime_LoadParams(TVMGraphRuntime* runtime, const char* param_blob, + const uint32_t param_size); + +/*! + * \brief Execute the graph. + * \param runtime The graph runtime. + */ +void TVMGraphRuntime_Run(TVMGraphRuntime* runtime); + +/*! + * \brief Release memory associated with the graph runtime. + * \param runtime Pointer to graph runtime. + */ +void TVMGraphRuntime_Release(TVMGraphRuntime** runtime); + +#ifdef __cplusplus +} // extern "C" +#endif + +#endif // TVM_RUNTIME_CRT_GRAPH_RUNTIME_H_ diff --git a/include/tvm/runtime/crt/memory.h b/include/tvm/runtime/crt/memory.h index 7b88b3123644..850c1ad69991 100644 --- a/include/tvm/runtime/crt/memory.h +++ b/include/tvm/runtime/crt/memory.h @@ -25,7 +25,13 @@ #ifndef TVM_RUNTIME_CRT_MEMORY_H_ #define TVM_RUNTIME_CRT_MEMORY_H_ -static int vleak_size = 0; +#ifdef __cplusplus +extern "C" { +#endif + +#include + +extern int vleak_size; /*! * \brief Allocate memory from manager @@ -49,4 +55,8 @@ void* vrealloc(void* ptr, size_t size); */ void vfree(void* ptr); +#ifdef __cplusplus +} // extern "C" +#endif + #endif // TVM_RUNTIME_CRT_MEMORY_H_ diff --git a/src/runtime/crt/module.h b/include/tvm/runtime/crt/module.h similarity index 70% rename from src/runtime/crt/module.h rename to include/tvm/runtime/crt/module.h index 57f8dd708f88..b825f6d55ed8 100644 --- a/src/runtime/crt/module.h +++ b/include/tvm/runtime/crt/module.h @@ -18,30 +18,24 @@ */ /*! - * \file src/runtime/crt/module.h + * \file include/tvm/runtime/crt/module.h * \brief Runtime container of the functions */ #ifndef TVM_RUNTIME_CRT_MODULE_H_ #define TVM_RUNTIME_CRT_MODULE_H_ -#include -#include - -struct TVMPackedFunc; +#include +#include /*! * \brief Module container of TVM. */ typedef struct TVMModule { - /*! - * \brief Get packed function from current module by name. - * - * \param name The name of the function. - * \param pf The result function. - * - * This function will return PackedFunc(nullptr) if function do not exist. - */ - void (*GetFunction)(struct TVMModule* mod, const char* name, struct TVMPackedFunc* pf); + /*! \brief The function registry associated with this mdoule. */ + const TVMFuncRegistry* registry; } TVMModule; +/*! \brief Entry point for the system lib module. */ +const TVMModule* TVMSystemLibEntryPoint(void); + #endif // TVM_RUNTIME_CRT_MODULE_H_ diff --git a/include/tvm/runtime/crt/packed_func.h b/include/tvm/runtime/crt/packed_func.h new file mode 100644 index 000000000000..0c39fe1a65b8 --- /dev/null +++ b/include/tvm/runtime/crt/packed_func.h @@ -0,0 +1,78 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +/*! + * \file tvm/runtime/crt/packed_func.h + * \brief Type-erased function used across TVM API. + */ +#ifndef TVM_RUNTIME_CRT_PACKED_FUNC_H_ +#define TVM_RUNTIME_CRT_PACKED_FUNC_H_ + +#include +#include +#include +#include +#include +#include + +#include "crt_config.h" + +DLDataType String2DLDataType(const char* s); + +typedef struct TVMArgs { + TVMValue values[TVM_CRT_MAX_ARGS]; + int tcodes[TVM_CRT_MAX_ARGS]; /* Data type should be identical to type_codes in TVMPackedCFunc */ + uint32_t values_count; +} TVMArgs; + +TVMArgs TVMArgs_Create(TVMValue* values, uint32_t* tcodes, uint32_t values_count); + +typedef struct TVMPackedFunc { + char name[200]; + TVMFunctionHandle fexec; + TVMArgs args; + TVMArgs ret_value; + int (*Call)(struct TVMPackedFunc* pf); + void (*SetArgs)(struct TVMPackedFunc* pf, const struct TVMArgs* args); +} TVMPackedFunc; + +int TVMPackedFunc_InitGlobalFunc(TVMPackedFunc* pf, const char* name, const TVMArgs* args); +int TVMPackedFunc_InitModuleFunc(TVMPackedFunc* pf, TVMModuleHandle module, const char* name, + const TVMArgs* args); + +int TVMPackedFunc_Call(TVMPackedFunc* pf); + +void TVMPackedFunc_SetArgs(TVMPackedFunc* pf, const TVMArgs* args); + +inline TVMModuleHandle TVMArgs_AsModuleHandle(const TVMArgs* args, size_t index) { + if (index >= args->values_count) { + TVMPlatformAbort(-1); + } + + if (args->tcodes[index] != kTVMModuleHandle) { + TVMPlatformAbort(-1); + } + + return args->values[index].v_handle; +} + +extern TVMPackedFunc* g_fexecs; +extern uint32_t g_fexecs_count; + +#endif // TVM_RUNTIME_CRT_PACKED_FUNC_H_ diff --git a/tests/cpp/crt_memory_test.cc b/include/tvm/runtime/crt/platform.h similarity index 51% rename from tests/cpp/crt_memory_test.cc rename to include/tvm/runtime/crt/platform.h index c2582ba02525..6897a53cfc1b 100644 --- a/tests/cpp/crt_memory_test.cc +++ b/include/tvm/runtime/crt/platform.h @@ -17,37 +17,28 @@ * under the License. */ -#define TVM_CRT_LOG_VIRT_MEM_SIZE 16 -#define TVM_CRT_PAGE_BYTES 4096 +/*! + * \file tvm/runtime/crt/platform.h + * \brief The virtual memory manager for micro-controllers + */ -#include -#include +#ifndef TVM_RUNTIME_CRT_PLATFORM_H_ +#define TVM_RUNTIME_CRT_PLATFORM_H_ -#include "../../src/runtime/crt/memory.c" +#ifdef __cplusplus +extern "C" { +#endif -TEST(CRTMemory, Alloc) { - for (int idx = 0; idx < 65536; idx++) { - void* a = vmalloc(1); - EXPECT_EQ(vleak_size, 1); - vfree(a); - EXPECT_EQ(vleak_size, 0); - } -} +/*! \brief Called when an internal error occurs and execution cannot continue. + * + * The platform should ideally restart or hang at this point. + * + * \param code An error code. + */ +void __attribute__((noreturn)) TVMPlatformAbort(int code); -TEST(CRTMemory, Realloc) { - for (int idx = 0; idx < 65536; idx++) { - void* a = vrealloc(0, 1); - EXPECT_EQ(vleak_size, 1); - void* b = vrealloc(a, 1); - EXPECT_EQ(a, b); - EXPECT_EQ(vleak_size, 1); - vfree(a); - EXPECT_EQ(vleak_size, 0); - } -} +#ifdef __cplusplus +} // extern "C" +#endif -int main(int argc, char** argv) { - testing::InitGoogleTest(&argc, argv); - testing::FLAGS_gtest_death_test_style = "threadsafe"; - return RUN_ALL_TESTS(); -} +#endif // TVM_RUNTIME_CRT_PLATFORM_H_ diff --git a/python/tvm/micro/func_registry.py b/python/tvm/micro/func_registry.py new file mode 100644 index 000000000000..c13a28ec1d72 --- /dev/null +++ b/python/tvm/micro/func_registry.py @@ -0,0 +1,76 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Defines functions to work with TVMModule FuncRegistry.""" + +import json + +def graph_json_to_c_func_registry(graph_path, func_registry_path): + """Convert a graph json file to a CRT-compatible FuncRegistry. + + Parameters + ---------- + graph_path : str + Path to the graph JSON file. + + func_registry_path : str + Path to a .c file which will be written containing the function registry. + """ + with open(graph_path) as json_f: + graph = json.load(json_f) + + funcs = [] + for n in graph['nodes']: + if n['op'] != 'tvm_op': + continue + + funcs.append(n['attrs']['func_name']) + + encoded_funcs = f'\\{len(funcs):03o}' + '\\0'.join(funcs) + lines = [ + '#include ', + '#include ', + '#include ', + '', + ] + + for f in funcs: + lines.append(f'extern int {f}(TVMValue* args, int* type_codes, int num_args, ' + 'TVMValue* out_ret_value, int* out_ret_tcode, void* resource_handle);') + + lines.append('static TVMBackendPackedCFunc funcs[] = {') + + for f in funcs: + lines.append(f' &{f},') + + lines += [ + '};', + 'static const TVMFuncRegistry system_lib_registry = {', + f' "{encoded_funcs}\\0",', + ' funcs,', + '};', + 'static const TVMModule system_lib = {', + ' &system_lib_registry,', + '};', + '', + 'const TVMModule* TVMSystemLibEntryPoint(void) {', + ' return &system_lib;', + '}', + '', # blank line to end the file + ] + with open(func_registry_path, 'w') as wrapper_f: + wrapper_f.write('\n'.join(lines)) diff --git a/src/runtime/crt/.gitignore b/src/runtime/crt/.gitignore new file mode 100644 index 000000000000..796b96d1c402 --- /dev/null +++ b/src/runtime/crt/.gitignore @@ -0,0 +1 @@ +/build diff --git a/src/runtime/crt/Makefile b/src/runtime/crt/Makefile new file mode 100644 index 000000000000..74c92696a3d9 --- /dev/null +++ b/src/runtime/crt/Makefile @@ -0,0 +1,57 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +ifeq ($(CRT_CONFIG),) +$(error "Must supply path to crt_config.h: CRT_CONFIG=...") +endif +DLPACK_INCLUDE_DIR ?= ../../../3rdparty/dlpack/include +TVM_INCLUDE_DIR ?= ../../../include + +BUILD_DIR ?= build +PREFIX ?= + +AR ?= ${PREFIX}ar +CC ?= ${PREFIX}gcc +RANLIB ?= ${PREFIX}ranlib + +QUIET ?= @ + +CFLAGS += -isystem "${TVM_INCLUDE_DIR}" -isystem "${DLPACK_INCLUDE_DIR}" -I include -I $(dir ${CRT_CONFIG}) +CFLAGS += -Werror -g +LDFLAGS += -Werror -g + +${BUILD_DIR}/%.o: %.c + ${QUIET}mkdir -p $(dir $@) + ${QUIET}${CC} ${CFLAGS} -c -o "$@" "$<" + +${BUILD_DIR}/common/libcommon.a: $(patsubst %.c,${BUILD_DIR}/%.o,$(wildcard common/*.c)) + ${QUIET}${AR} -cr "$@" $^ + ${QUIET}${RANLIB} ${RANLIBFLAGS} "$@" + +${BUILD_DIR}/graph_runtime/libgraph_runtime.a: $(patsubst %.c,${BUILD_DIR}/%.o,$(wildcard graph_runtime/*.c)) + ${QUIET}${AR} -cr "$@" $^ + ${QUIET}${RANLIB} ${RANLIBFLAGS} "$@" + +common: ${BUILD_DIR}/common/libcommon.a +graph_runtime: ${BUILD_DIR}/graph_runtime/libgraph_runtime.a + +all: common graph_runtime +clean: + rm -rf "${BUILD_DIR}" + +.PHONY: all common graph_runtime +.DEFAULT_GOAL: all diff --git a/src/runtime/crt/crt_backend_api.c b/src/runtime/crt/common/crt_backend_api.c similarity index 85% rename from src/runtime/crt/crt_backend_api.c rename to src/runtime/crt/common/crt_backend_api.c index 7589ce479014..2e418ca67c6b 100644 --- a/src/runtime/crt/crt_backend_api.c +++ b/src/runtime/crt/common/crt_backend_api.c @@ -17,15 +17,16 @@ * under the License. */ +// LINT_C_FILE + #include #include #include #include #include +#include #include -#include "packed_func.h" - void* TVMBackendAllocWorkspace(int device_type, int device_id, uint64_t nbytes, int dtype_code_hint, int dtype_bits_hint) { void* ptr = 0; @@ -48,9 +49,5 @@ int TVMBackendParallelLaunch(FTVMParallelLambda flambda, void* cdata, int num_ta } int TVMBackendRegisterSystemLibSymbol(const char* name, void* ptr) { - g_fexecs = vrealloc(g_fexecs, sizeof(TVMPackedFunc) * (g_fexecs_count + 1)); - snprintf(g_fexecs[g_fexecs_count].name, sizeof(g_fexecs[g_fexecs_count].name), "%s", name); - g_fexecs[g_fexecs_count].fexec = ptr; - g_fexecs_count++; - return 0; + return TVMFuncRegisterGlobal(name, ptr, 0); } diff --git a/src/runtime/crt/common/crt_runtime_api.c b/src/runtime/crt/common/crt_runtime_api.c new file mode 100644 index 000000000000..12b74db42f2a --- /dev/null +++ b/src/runtime/crt/common/crt_runtime_api.c @@ -0,0 +1,335 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +// LINT_C_FILE + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +// Handle internal errors + +static char g_last_error[1024]; + +void TVMAPISetLastError(const char* msg) { strncpy(g_last_error, msg, sizeof(g_last_error)); } + +__attribute__((format(printf, 1, 2))) int TVMAPIErrorf(const char* msg, ...) { + va_list args; + int to_return; + + va_start(args, msg); + to_return = vsnprintf(g_last_error, sizeof(g_last_error), msg, args); + va_end(args); + + return to_return; +} + +const char* TVMGetLastError(void) { return g_last_error; } + +// Manipulate NDArray on target device + +int TVMArrayAlloc(const tvm_index_t* shape, int ndim, int dtype_code, int dtype_bits, + int dtype_lanes, int device_type, int device_id, TVMArrayHandle* out) { + DLDataType dtype; + dtype.code = dtype_code; + dtype.bits = dtype_bits; + dtype.lanes = dtype_lanes; + DLContext ctx; + ctx.device_type = (DLDeviceType)device_type; + ctx.device_id = device_id; + TVMNDArray arr = TVMNDArray_Empty(ndim, shape, dtype, ctx); + **out = arr.dl_tensor; + return 0; +} + +int TVMArrayFree(TVMArrayHandle handle) { + TVMNDArray arr; + arr.dl_tensor = *handle; + return TVMNDArray_Release(&arr); +} + +int TVMDeviceAllocDataSpace(DLContext ctx, size_t nbytes, size_t alignment, DLDataType type_hint, + void** out_data) { + if (alignment != 1) { + nbytes = (nbytes + alignment - 1) / alignment * alignment; + } + + *out_data = vmalloc(nbytes); + return 0; +} + +int TVMDeviceFreeDataSpace(TVMContext ctx, void* ptr) { + vfree(ptr); + return 0; +} + +int TVMDeviceCopyDataFromTo(const void* from, size_t from_offset, void* to, size_t to_offset, + size_t num_bytes, TVMContext ctx_from, TVMContext ctx_to, + DLDataType type_hint, TVMStreamHandle stream) { + memcpy(((uint8_t*)to) + to_offset, ((uint8_t*)from) + from_offset, num_bytes); + return 0; +} + +int TVMSynchronize(int device_type, int device_id, TVMStreamHandle stream) { return 0; } + +static TVMMutableFuncRegistry global_func_registry; + +int TVMFuncRegisterGlobal(const char* name, TVMFunctionHandle f, int override) { + return TVMMutableFuncRegistry_Set(&global_func_registry, name, f, override != 0); +} + +static const TVMModule* registered_modules[TVM_CRT_MAX_REGISTERED_MODULES]; + +/*! \brief Passed as `module_index` to EncodeFunctionHandle. */ +static const tvm_module_index_t kGlobalFuncModuleIndex = TVM_CRT_MAX_REGISTERED_MODULES; + +static int DecodeModuleHandle(TVMModuleHandle handle, tvm_module_index_t* out_module_index) { + tvm_module_index_t module_index; + + module_index = ((tvm_module_index_t)((uintptr_t)handle)) & ~0x8000; + if (module_index > TVM_CRT_MAX_REGISTERED_MODULES || registered_modules[module_index] == NULL) { + TVMAPIErrorf("invalid module handle: %08x", module_index); + return -1; + } + + *out_module_index = module_index; + return 0; +} + +static TVMModuleHandle EncodeModuleHandle(tvm_module_index_t module_index) { + return (TVMModuleHandle)((uintptr_t)(module_index | 0x8000)); +} + +static int TVMModCreateFromCModule(const TVMModule* mod, TVMModuleHandle* out_handle) { + tvm_module_index_t idx; + + for (idx = 0; idx < TVM_CRT_MAX_REGISTERED_MODULES; idx++) { + if (registered_modules[idx] == NULL) { + registered_modules[idx] = mod; + *out_handle = EncodeModuleHandle(idx); + return 0; + } + } + + return -1; +} + +int TVMModFree(TVMModuleHandle mod) { + tvm_module_index_t module_index; + if (DecodeModuleHandle(mod, &module_index) != 0) { + return -1; + } + + registered_modules[module_index] = NULL; + return 0; +} + +static const TVMModuleHandle kTVMModuleHandleUninitialized = (TVMModuleHandle)(~0UL); + +static TVMModuleHandle system_lib_handle; + +int SystemLibraryCreate(TVMValue* args, int* type_codes, int num_args, TVMValue* ret_val, + int* ret_type_codes) { + const TVMModule* system_lib; + + if (system_lib_handle == kTVMModuleHandleUninitialized) { + system_lib = TVMSystemLibEntryPoint(); + if (TVMModCreateFromCModule(system_lib, &system_lib_handle) != 0) { + TVMAPIErrorf("error registering system lib"); + return -1; + } + } + + ret_val[0].v_handle = system_lib_handle; + ret_type_codes[0] = kTVMModuleHandle; + return 0; +} + +static TVMFunctionHandle EncodeFunctionHandle(tvm_module_index_t module_index, + tvm_function_index_t function_index) { + return (TVMFunctionHandle)((uintptr_t)( + ((module_index | 0x8000) << (sizeof(tvm_function_index_t) * 8)) | (function_index | 0x8000))); +} + +static int DecodeFunctionHandle(TVMFunctionHandle handle, tvm_module_index_t* module_index, + tvm_function_index_t* function_index) { + tvm_module_index_t unvalidated_module_index; + unvalidated_module_index = + (tvm_module_index_t)(((uintptr_t)handle) >> (sizeof(tvm_function_index_t) * 8)); + unvalidated_module_index &= ~0x8000; + + if (unvalidated_module_index > kGlobalFuncModuleIndex) { + TVMAPIErrorf("invalid module handle: index=%08x", unvalidated_module_index); + return -1; + } else if (unvalidated_module_index < kGlobalFuncModuleIndex && + registered_modules[unvalidated_module_index] == NULL) { + TVMAPIErrorf("unregistered module: index=%08x", unvalidated_module_index); + return -1; + } + + *function_index = ((uint32_t)((uintptr_t)handle)) & ~0x8000; + *module_index = unvalidated_module_index; + return 0; +} + +int TVMFuncCall(TVMFunctionHandle func_handle, TVMValue* arg_values, int* type_codes, int num_args, + TVMValue* ret_val, int* ret_type_code) { + tvm_module_index_t module_index; + tvm_function_index_t function_index; + void* resource_handle; + const TVMFuncRegistry* registry; + TVMBackendPackedCFunc func; + + if (DecodeFunctionHandle(func_handle, &module_index, &function_index) != 0) { + return -1; + } + + if (module_index == kGlobalFuncModuleIndex) { + resource_handle = NULL; + registry = &global_func_registry.registry; + } else { + resource_handle = (void*)registered_modules[module_index]->registry; + registry = registered_modules[module_index]->registry; + } + + if (TVMFuncRegistry_GetByIndex(registry, function_index, &func) != 0) { + TVMAPIErrorf("invalid function index: %04" PRIx16, function_index); + return -1; + } + + ret_type_code[0] = kTVMNullptr; + ret_val[0].v_handle = NULL; + return func(arg_values, type_codes, num_args, ret_val, ret_type_code, resource_handle); +} + +static int FindFunctionOrSetAPIError(tvm_module_index_t module_index, + const TVMFuncRegistry* registry, const char* name, + TVMFunctionHandle* out) { + tvm_function_index_t function_index; + if (TVMFuncRegistry_Lookup(registry, name, &function_index) != 0) { + TVMAPIErrorf("failed to get function: mod_index=%04" PRIx16 ", name=%s", module_index, name); + return -1; + } + + *out = EncodeFunctionHandle(module_index, function_index); + return 0; +} + +int TVMFuncGetGlobal(const char* name, TVMFunctionHandle* out) { + return FindFunctionOrSetAPIError(kGlobalFuncModuleIndex, &global_func_registry.registry, name, + out); +} + +int TVMModGetFunction(TVMModuleHandle mod, const char* func_name, int query_imports, + TVMFunctionHandle* out) { + tvm_module_index_t module_index; + if (DecodeModuleHandle(mod, &module_index) != 0) { + return -1; + } + + return FindFunctionOrSetAPIError(module_index, registered_modules[module_index]->registry, + func_name, out); +} + +int ModuleGetFunction(TVMValue* args, int* type_codes, int num_args, TVMValue* ret_value, + int* ret_type_codes) { + int function_index; + TVMModuleHandle mod; + int module_index; + const char* name; + int to_return; + int query_imports; + + ret_value[0].v_handle = NULL; + ret_type_codes[0] = kTVMNullptr; + if (num_args != 3 || type_codes[0] != kTVMModuleHandle || type_codes[1] != kTVMStr || + type_codes[2] != kDLInt) { + return 0; + } + + mod = (TVMModuleHandle)args[0].v_handle; + name = args[1].v_str; + query_imports = args[2].v_int64 != 0; + to_return = TVMModGetFunction(mod, name, query_imports, &ret_value->v_handle); + + if (to_return == 0) { + ret_type_codes[0] = kTVMPackedFuncHandle; + } + + return to_return; +} + +typedef struct TVMCReturnValue { + TVMValue* ret_val; + int* ret_type_code; +} TVMCReturnValue; + +int TVMCFuncSetReturn(TVMRetValueHandle ret, TVMValue* value, int* type_code, int num_ret) { + TVMCReturnValue* ret_val; + int idx; + + ret_val = (TVMCReturnValue*)ret; + for (idx = 0; idx < num_ret; idx++) { + ret_val->ret_val[idx] = value[idx]; + ret_val->ret_type_code[idx] = type_code[idx]; + } + + return 0; +} + +int TVMFuncFree(TVMFunctionHandle func) { + // A no-op, since we don't actually allocate anything in GetFunction + return 0; +} + +tvm_crt_error_t TVMInitializeRuntime() { + int idx; + int error; + + system_lib_handle = kTVMModuleHandleUninitialized; + + TVMMutableFuncRegistry_Create(&global_func_registry, + vmalloc(TVM_CRT_GLOBAL_FUNC_REGISTRY_SIZE_BYTES), + TVM_CRT_GLOBAL_FUNC_REGISTRY_SIZE_BYTES); + for (idx = 0; idx < TVM_CRT_MAX_REGISTERED_MODULES; idx++) { + registered_modules[idx] = NULL; + } + + error = TVMFuncRegisterGlobal("runtime.SystemLib", &SystemLibraryCreate, 0); + if (error != 0) { + return error; + } + + error = TVMFuncRegisterGlobal("tvm.rpc.server.ModuleGetFunction", &ModuleGetFunction, 0); + if (error != 0) { + return error; + } + + return 0; +} diff --git a/src/runtime/crt/common/func_registry.c b/src/runtime/crt/common/func_registry.c new file mode 100644 index 000000000000..1ffffa5eaf62 --- /dev/null +++ b/src/runtime/crt/common/func_registry.c @@ -0,0 +1,152 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +// LINT_C_FILE + +/*! + * \file tvm/runtime/crt/func_registry.c + * \brief Defines implementations of generic string-based function lookup structs + */ + +#include +#include +#include + +/*! + * \brief strcmp against the next string in the registry, and return the end. + * + * Regardless of return value, after calling this function, cursor's value will be modified to + * point at the \0 at the end of the string it currently points to. + * + * \param cursor Pointer to cursor to first string to compare. + * \param name Pointer to reference string. + * \return 0 if the string pointed to by cursor == name; non-zero otherwise. + */ +int strcmp_cursor(const char** cursor, const char* name) { + int return_value = 0; + while (return_value == 0) { + char c = **cursor; + char n = *name; + return_value = ((int)c) - ((int)n); + + if (n == 0 || c == 0) { + break; + } + + name++; + (*cursor)++; + } + + while (**cursor != 0) { + (*cursor)++; + } + + return return_value; +} + +tvm_crt_error_t TVMFuncRegistry_Lookup(const TVMFuncRegistry* reg, const char* name, + tvm_function_index_t* function_index) { + tvm_function_index_t idx; + const char* reg_name_ptr; + + idx = 0; + // NOTE: reg_name_ptr starts at index 1 to skip num_funcs. + for (reg_name_ptr = reg->names + 1; *reg_name_ptr != '\0'; reg_name_ptr++) { + if (!strcmp_cursor(®_name_ptr, name)) { + *function_index = idx; + return kTvmErrorNoError; + } + + idx++; + } + + return kTvmErrorFunctionNameNotFound; +} + +tvm_crt_error_t TVMFuncRegistry_GetByIndex(const TVMFuncRegistry* reg, + tvm_function_index_t function_index, + TVMBackendPackedCFunc* out_func) { + uint8_t num_funcs; + + num_funcs = reg->names[0]; + if (function_index >= num_funcs) { + return kTvmErrorFunctionIndexInvalid; + } + + *out_func = reg->funcs[function_index]; + return kTvmErrorNoError; +} + +tvm_crt_error_t TVMMutableFuncRegistry_Create(TVMMutableFuncRegistry* reg, uint8_t* buffer, + size_t buffer_size_bytes) { + if (buffer_size_bytes < kTvmAverageFuncEntrySizeBytes) { + return kTvmErrorBufferTooSmall; + } + + memset(reg, 0, sizeof(*reg)); + reg->registry.names = (const char*)buffer; + buffer[0] = 0; // number of functions present in buffer. + buffer[1] = 0; // end of names list marker. + + // compute a guess of the average size of one entry: + // - assume average function name is around ~10 bytes + // - 1 byte for \0 + // - size of 1 function pointer + reg->max_functions = buffer_size_bytes / kTvmAverageFuncEntrySizeBytes; + reg->registry.funcs = + (TVMBackendPackedCFunc*)(buffer + buffer_size_bytes - reg->max_functions * sizeof(void*)); + + return kTvmErrorNoError; +} + +tvm_crt_error_t TVMMutableFuncRegistry_Set(TVMMutableFuncRegistry* reg, const char* name, + TVMBackendPackedCFunc func, int override) { + size_t idx; + char* reg_name_ptr; + + idx = 0; + // NOTE: safe to discard const qualifier here, since reg->registry.names was set from + // TVMMutableFuncRegistry_Create above. + // NOTE: reg_name_ptr starts at index 1 to skip num_funcs. + for (reg_name_ptr = (char*)reg->registry.names + 1; *reg_name_ptr != 0; reg_name_ptr++) { + if (!strcmp_cursor((const char**)®_name_ptr, name)) { + if (override == 0) { + return kTvmErrorFunctionAlreadyDefined; + } + ((TVMBackendPackedCFunc*)reg->registry.funcs)[idx] = func; + return kTvmErrorNoError; + } + + idx++; + } + + size_t name_len = strlen(name); + ssize_t names_bytes_remaining = ((const char*)reg->registry.funcs) - reg_name_ptr; + if (idx >= reg->max_functions || name_len + 1 > names_bytes_remaining) { + return kTvmErrorFunctionRegistryFull; + } + + memcpy(reg_name_ptr, name, name_len + 1); + reg_name_ptr += name_len + 1; + *reg_name_ptr = 0; + ((TVMBackendPackedCFunc*)reg->registry.funcs)[idx] = func; + ((char*)reg->registry.names)[0]++; // increment num_funcs. + + return kTvmErrorNoError; +} diff --git a/src/runtime/crt/memory.c b/src/runtime/crt/common/memory.c similarity index 52% rename from src/runtime/crt/memory.c rename to src/runtime/crt/common/memory.c index c25749e44493..4ede8ca45783 100644 --- a/src/runtime/crt/memory.c +++ b/src/runtime/crt/common/memory.c @@ -17,103 +17,65 @@ * under the License. */ +// LINT_C_FILE + /*! * \file memory.c - * \brief Virtal memory manager + * \brief Virtual memory manager * * To maximize portability, thread-safe feature has been dropped for now. */ #include +#include +#include +#include +#include #include +#include +#include #include - -#include "logging.h" - -/*! Number of bits in a page */ -#define TVM_CRT_PAGE_BITS (TVM_CRT_PAGE_BYTES << 3) - -/*! \brief Translate log memory size into bytes */ -#define TVM_CRT_VIRT_MEM_SIZE (1 << TVM_CRT_LOG_VIRT_MEM_SIZE) - -/*! \brief Number of possible page entries in total */ -#define TVM_CRT_MAX_PAGES (TVM_CRT_VIRT_MEM_SIZE / TVM_CRT_PAGE_BYTES) - -/*! \brief Physical address type */ -typedef uint32_t tvm_phy_addr_t; - -/*! \brief The bits in page table */ -static const tvm_phy_addr_t kPageBits = TVM_CRT_PAGE_BITS; - -/*! \brief Page size, also the maximum allocable size */ -static const tvm_phy_addr_t kPageSize = TVM_CRT_PAGE_BYTES; +#include /** * \brief Memory pool for virtual dynamic memory allocation */ -static char g_memory_pool[TVM_CRT_VIRT_MEM_SIZE]; - -/*! \brief A page in the DRAM */ -typedef struct Page { - /*! \brief Start location in page table */ - tvm_index_t ptable_begin; - /*! \brief The total number of pages */ - tvm_index_t num_pages; - /*! \brief Data */ - char* data; -} Page; +static uint8_t g_memory_pool[TVM_CRT_VIRT_MEM_SIZE]; // construct a new page -Page PageCreate(tvm_index_t ptable_begin, tvm_index_t num_pages) { +Page PageCreate(uint8_t* memory_pool, size_t page_size_bytes, tvm_index_t ptable_begin, + tvm_index_t num_pages) { Page page; page.ptable_begin = ptable_begin; page.num_pages = num_pages; - page.data = g_memory_pool + ptable_begin * kPageSize; + page.data = memory_pool + ptable_begin * page_size_bytes; return page; } -typedef struct PageTable { - Page page[TVM_CRT_MAX_PAGES]; - uint32_t count; - void (*resize)(struct PageTable* ptable, uint32_t size, Page* page); -} PageTable; - -void PageTable_Resize(struct PageTable* ptable, uint32_t new_size, Page* page) { - CHECK_LE(ptable->count, new_size, "size value (%d) is smaller than expected (%d).", new_size, - ptable->count); - for (uint32_t idx = ptable->count; idx < new_size; idx++) { +void PageTable_Resize(struct PageTable* ptable, size_t new_size, Page* page) { + CHECK_LE(ptable->num_pages, new_size, "size value (%zu) is smaller than expected (%zu).", + new_size, ptable->num_pages); + for (uint32_t idx = ptable->num_pages; idx < new_size; idx++) { ptable->page[idx] = *page; } - ptable->count = new_size; + ptable->num_pages = new_size; } -typedef struct PageEntry { - char* addr; - Page page; -} PageEntry; - -typedef struct TLB { - PageEntry entries[TVM_CRT_MAX_PAGES]; - uint32_t count; - void (*set)(struct TLB* tlb, char* data, Page* page); - PageEntry* (*find)(struct TLB* tlb, char* data); -} TLB; - -void TLB_Set(TLB* tlb, char* data, Page* page) { +void TLB_Set(TLB* tlb, uint8_t* data, Page* page) { PageEntry* entry = tlb->find(tlb, data); if (entry == 0) { - tlb->entries[tlb->count].addr = data; - tlb->entries[tlb->count].page = *page; - tlb->count++; + tlb->entries[tlb->num_pages].addr = data; + tlb->entries[tlb->num_pages].page = *page; + tlb->num_pages++; } else { entry->addr = data; entry->page = *page; } } -PageEntry* TLB_Find(TLB* tlb, char* data) { +PageEntry* TLB_Find(TLB* tlb, uint8_t* data) { PageEntry* entry = 0; - for (uint32_t idx = 0; idx < tlb->count; idx++) { + for (uint32_t idx = 0; idx < tlb->num_pages; idx++) { if (tlb->entries[idx].addr == data) { entry = tlb->entries + idx; break; @@ -122,23 +84,9 @@ PageEntry* TLB_Find(TLB* tlb, char* data) { return entry; } -typedef struct IndexedEntry { - tvm_index_t index; - Page page; -} IndexedEntry; - -typedef struct MultiMap { - IndexedEntry entries[TVM_CRT_MAX_PAGES]; - uint32_t count; - IndexedEntry* (*lower_bound)(struct MultiMap* map, uint32_t npage); - IndexedEntry* (*end)(struct MultiMap* map); - void (*erase)(struct MultiMap* map, IndexedEntry* entry); - void (*insert)(struct MultiMap* map, uint32_t npage, Page* p); -} MultiMap; - IndexedEntry* MultiMap_LowerBound(struct MultiMap* map, uint32_t npage) { IndexedEntry* entry = 0; - for (uint32_t idx = 0; idx < map->count; idx++) { + for (uint32_t idx = 0; idx < map->num_entries; idx++) { if (map->entries[idx].index >= npage) { entry = map->entries + idx; break; @@ -153,66 +101,37 @@ IndexedEntry* MultiMap_End(struct MultiMap* map) { } void MultiMap_Erase(struct MultiMap* map, IndexedEntry* entry) { - for (uint32_t idx = 0; idx < map->count; idx++) { + for (uint32_t idx = 0; idx < map->num_entries; idx++) { if ((map->entries + idx) == entry) { - memcpy(map->entries + idx, map->entries + (idx + 1), - sizeof(IndexedEntry) * (map->count - idx)); - map->count--; + // NOTE: do not use memcpy due to overlap. + for (uint32_t src_idx = idx + 1; src_idx < map->num_entries; src_idx++) { + map->entries[src_idx - 1] = map->entries[src_idx]; + } + map->num_entries--; break; } } } void MultiMap_Insert(struct MultiMap* map, uint32_t npage, Page* p) { - CHECK_LE(map->count + 1, TVM_CRT_MAX_PAGES, "invalid number of free pages."); - for (uint32_t idx = map->count; idx < (map->count + npage); idx++) { - map->entries[map->count].index = npage; - map->entries[map->count].page = *p; + CHECK_LE(map->num_entries + 1, map->max_entries, "invalid number of free pages."); + for (uint32_t idx = map->num_entries; idx < (map->num_entries + npage); idx++) { + map->entries[map->num_entries].index = npage; + map->entries[map->num_entries].page = *p; } - map->count++; + map->num_entries++; } -/*! - * \brief DRAM memory manager - * Implements simple paging to allow physical address translation. - */ -typedef struct MemoryManager { - /*! - * \brief Allocate memory from manager - * \param size The size of memory - * \return The virtual address - */ - void* (*Alloc)(struct MemoryManager* mgr, tvm_index_t size); - /*! - * \brief Allocate memory from manager - * \param ptr The pointer to the memory area to be reallocated - * \param size The size of memory - * \return The virtual address - */ - void* (*Realloc)(struct MemoryManager* mgr, void* ptr, tvm_index_t size); - /*! - * \brief Free the memory. - * \param ptr The pointer to the memory to deallocate - * \return The virtual address - */ - void (*Free)(struct MemoryManager* mgr, void* data); - - // Physical address -> page - PageTable ptable; - // Virtual address -> page - TLB pmap; - // Free map - MultiMap free_map; -} MemoryManager; - /*! * \brief Allocate memory from manager * \param size The size of memory * \return The virtual address */ void* MemoryManager_Alloc(MemoryManager* mgr, tvm_index_t size) { - char* data = 0; - tvm_index_t npage = (size + kPageSize - 1) / kPageSize; + uint8_t* data = 0; + PageTable* ptable = &(mgr->ptable); + tvm_index_t npage = (size + ptable->page_size_bytes - 1) / ptable->page_size_bytes; + MultiMap* free_map = &(mgr->free_map); IndexedEntry* it = free_map->lower_bound(free_map, npage); tvm_index_t start = 0; @@ -223,13 +142,12 @@ void* MemoryManager_Alloc(MemoryManager* mgr, tvm_index_t size) { start = p.ptable_begin; npage = p.num_pages; } else { - PageTable* ptable = &(mgr->ptable); - start = ptable->count; - CHECK_LE((unsigned)(start + npage), (sizeof(g_memory_pool) / kPageSize), + start = ptable->num_pages; + CHECK_LE((unsigned)(start + npage), ptable->max_pages, "insufficient memory, start=%" PRId64 ", npage=%" PRId64 ", total=%" PRId64 "", start, npage, start + npage); /* insert page entry */ - Page p = PageCreate(start, npage); + Page p = PageCreate(ptable->memory_pool, ptable->page_size_bytes, start, npage); ptable->resize(ptable, start + npage, &p); data = p.data; TLB* pmap = &(mgr->pmap); @@ -237,8 +155,8 @@ void* MemoryManager_Alloc(MemoryManager* mgr, tvm_index_t size) { } vleak_size++; #if TVM_CRT_DEBUG > 1 - printf("allocate: addr=%p, start=%d/%d, npage=%d, vleak=%d\n", data, start, TVM_CRT_MAX_PAGES, - npage, vleak_size); + printf("allocate: addr=%p, start=%" PRId64 "/%zu, npage=%" PRId64 ", vleak=%d\n", data, start, + ptable->max_pages, npage, vleak_size); #endif // TVM_CRT_DEBUG return data; } @@ -250,16 +168,16 @@ void* MemoryManager_Alloc(MemoryManager* mgr, tvm_index_t size) { * \return The virtual address */ void* MemoryManager_Realloc(MemoryManager* mgr, void* ptr, tvm_index_t size) { - char* data = (char*)ptr; // NOLINT(*) + uint8_t* data = (uint8_t*)ptr; // NOLINT(*) PageTable* ptable = &(mgr->ptable); TLB* pmap = &(mgr->pmap); MultiMap* free_map = &(mgr->free_map); tvm_index_t start = 0; - tvm_index_t npage = (size + kPageSize - 1) / kPageSize; + tvm_index_t npage = (size + ptable->page_size_bytes - 1) / ptable->page_size_bytes; if (ptr) { // get page size for given pointer - CHECK_NE(pmap->count, 0, "invalid translation look-aside buffer."); - PageEntry* entry = pmap->find(pmap, (char*)ptr); // NOLINT(*) + CHECK_NE(pmap->num_pages, 0, "invalid translation look-aside buffer."); + PageEntry* entry = pmap->find(pmap, (uint8_t*)ptr); // NOLINT(*) CHECK_NE(entry, 0, "no valid page entry found."); Page* pptr = &(entry->page); // if the page size is smaller than target page size, @@ -275,17 +193,17 @@ void* MemoryManager_Realloc(MemoryManager* mgr, void* ptr, tvm_index_t size) { npage = it->page.num_pages; free_map->erase(free_map, it); } else { - start = ptable->count; - CHECK_LE((unsigned)(start + npage), (sizeof(g_memory_pool) / kPageSize), + start = ptable->num_pages; + CHECK_LE((unsigned)(start + npage), ptable->max_pages, "insufficient memory, start=%" PRId64 ", npage=%" PRId64 ", total=%" PRId64 "", start, npage, start + npage); - Page p = PageCreate(start, npage); + Page p = PageCreate(mgr->ptable.memory_pool, mgr->ptable.page_size_bytes, start, npage); ptable->resize(ptable, start + npage, &p); data = p.data; pmap->set(pmap, data, &p); } // copy previous data to the new entry - memcpy(data, ptr, kPageSize * pptr->num_pages); + memcpy(data, ptr, ptable->page_size_bytes * pptr->num_pages); // release memory free_map->insert(free_map, pptr->num_pages, pptr); } else { @@ -301,12 +219,12 @@ void* MemoryManager_Realloc(MemoryManager* mgr, void* ptr, tvm_index_t size) { npage = p.num_pages; } else { PageTable* ptable = &(mgr->ptable); - start = ptable->count; - CHECK_LE((unsigned)(start + npage), (sizeof(g_memory_pool) / kPageSize), + start = ptable->num_pages; + CHECK_LE((unsigned)(start + npage), ptable->max_pages, "insufficient memory, start=%" PRId64 ", npage=%" PRId64 ", total=%" PRId64 "", start, npage, start + npage); /* insert page entry */ - Page p = PageCreate(start, npage); + Page p = PageCreate(mgr->ptable.memory_pool, mgr->ptable.page_size_bytes, start, npage); ptable->resize(ptable, start + npage, &p); data = p.data; TLB* pmap = &(mgr->pmap); @@ -315,8 +233,9 @@ void* MemoryManager_Realloc(MemoryManager* mgr, void* ptr, tvm_index_t size) { vleak_size++; } #if TVM_CRT_DEBUG > 1 - printf("reallocate: addr=%p, start=%d/%d, npage=%d, vleak=%d, size=%d\n", data, start, - TVM_CRT_MAX_PAGES, npage, vleak_size, size); + printf("reallocate: addr=%p, start=%" PRId64 "/%zu, npage=%" PRId64 ", vleak=%d, size=%" PRId64 + "\n", + data, start, mgr->ptable.max_pages, npage, vleak_size, size); #endif // TVM_CRT_DEBUG return data; } @@ -328,49 +247,79 @@ void* MemoryManager_Realloc(MemoryManager* mgr, void* ptr, tvm_index_t size) { */ void MemoryManager_Free(MemoryManager* mgr, void* ptr) { TLB* pmap = &(mgr->pmap); - CHECK_NE(pmap->count, 0, "invalid translation look-aside buffer."); - PageEntry* entry = pmap->find(pmap, (char*)ptr); // NOLINT(*) + CHECK_NE(pmap->num_pages, 0, "invalid translation look-aside buffer."); + PageEntry* entry = pmap->find(pmap, (uint8_t*)ptr); // NOLINT(*) CHECK_NE(entry, 0, "no valid page entry found."); Page* p = &(entry->page); MultiMap* free_map = &(mgr->free_map); free_map->insert(free_map, p->num_pages, p); vleak_size--; #if TVM_CRT_DEBUG > 1 - printf("release: addr=%p, start=%d/%d, npage=%d, vleak=%d\n", ptr, entry->page.ptable_begin, - TVM_CRT_MAX_PAGES, entry->page.num_pages, vleak_size); + printf("release: addr=%p, start=%" PRId64 "/%zu, npage=%" PRId64 ", vleak=%d\n", ptr, + entry->page.ptable_begin, mgr->ptable.max_pages, entry->page.num_pages, vleak_size); #endif // TVM_CRT_DEBUG } -MemoryManager* MemoryManagerCreate() { - static MemoryManager mgr; - memset(&mgr, 0, sizeof(MemoryManager)); +#define ROUND_UP(qty, modulo) (((qty) + ((modulo)-1)) / (modulo) * (modulo)) + +void MemoryManagerCreate(MemoryManager* manager, uint8_t* memory_pool, + size_t memory_pool_size_bytes, size_t page_size_bytes_log2) { + memset(manager, 0, sizeof(MemoryManager)); + memset(memory_pool, 0, sizeof(memory_pool_size_bytes)); + /* handle MemoryManager member functions */ - mgr.Alloc = MemoryManager_Alloc; - mgr.Realloc = MemoryManager_Realloc; - mgr.Free = MemoryManager_Free; + manager->Alloc = MemoryManager_Alloc; + manager->Realloc = MemoryManager_Realloc; + manager->Free = MemoryManager_Free; + + // Allocate enough space for MAX_PAGES. + size_t page_size_bytes = 1 << page_size_bytes_log2; + size_t metadata_bytes_per_page = sizeof(Page) + sizeof(PageEntry) + sizeof(IndexedEntry); + size_t bytes_needed_per_page = page_size_bytes + metadata_bytes_per_page; + size_t num_pages = memory_pool_size_bytes / bytes_needed_per_page; + + size_t metadata_pages_bytes = ROUND_UP(metadata_bytes_per_page * num_pages, page_size_bytes); + size_t metadata_num_pages = metadata_pages_bytes >> page_size_bytes_log2; + uint8_t* metadata_cursor = memory_pool + (num_pages << page_size_bytes_log2); + + manager->ptable.memory_pool = memory_pool; + /* handle PageTable member functions */ - mgr.ptable.resize = PageTable_Resize; + manager->ptable.page = (Page*)metadata_cursor; + metadata_cursor += sizeof(Page) * num_pages; + + manager->ptable.page_size_bytes = (1 << page_size_bytes_log2); + manager->ptable.max_pages = num_pages; + manager->ptable.resize = PageTable_Resize; + /* handle TLB member functions */ - mgr.pmap.set = TLB_Set; - mgr.pmap.find = TLB_Find; + manager->pmap.entries = (PageEntry*)metadata_cursor; + metadata_cursor += sizeof(PageEntry) * num_pages; + manager->pmap.max_pages = num_pages; + manager->pmap.num_pages = 0; + + manager->pmap.set = TLB_Set; + manager->pmap.find = TLB_Find; /* handle free_map member functions */ - mgr.free_map.lower_bound = MultiMap_LowerBound; - mgr.free_map.end = MultiMap_End; - mgr.free_map.erase = MultiMap_Erase; - mgr.free_map.insert = MultiMap_Insert; - return &mgr; + manager->free_map.entries = (IndexedEntry*)metadata_cursor; + metadata_cursor += sizeof(IndexedEntry) * num_pages; + manager->free_map.max_entries = num_pages; + manager->free_map.lower_bound = MultiMap_LowerBound; + manager->free_map.end = MultiMap_End; + manager->free_map.erase = MultiMap_Erase; + manager->free_map.insert = MultiMap_Insert; } MemoryManager* TVMGetGlobalMemoryManager() { /* initialize once */ static uint32_t initialized = 0; - static MemoryManager* mgr; + static MemoryManager mgr; if (!initialized) { - mgr = MemoryManagerCreate(); memset(g_memory_pool, 0, sizeof(g_memory_pool)); + MemoryManagerCreate(&mgr, g_memory_pool, TVM_CRT_VIRT_MEM_SIZE, TVM_CRT_PAGE_BYTES_LOG); initialized = 1; } - return mgr; + return &mgr; } /** \brief Allocate memory from manager */ @@ -390,3 +339,5 @@ void vfree(void* ptr) { MemoryManager* mgr = TVMGetGlobalMemoryManager(); mgr->Free(mgr, ptr); } + +int vleak_size = 0; diff --git a/src/runtime/crt/ndarray.c b/src/runtime/crt/common/ndarray.c similarity index 97% rename from src/runtime/crt/ndarray.c rename to src/runtime/crt/common/ndarray.c index 17e210785aa1..f16db693b76f 100644 --- a/src/runtime/crt/ndarray.c +++ b/src/runtime/crt/common/ndarray.c @@ -17,15 +17,18 @@ * under the License. */ +// LINT_C_FILE + /*! * \file ndarray.c * \brief NDArray container infratructure. */ -#include "ndarray.h" - +#include #include +#include "crt_config.h" + TVMNDArray TVMNDArray_Create(uint32_t ndim, const tvm_index_t* shape, DLDataType dtype, DLContext ctx) { TVMNDArray ret; diff --git a/src/runtime/crt/packed_func.h b/src/runtime/crt/common/packed_func.c similarity index 52% rename from src/runtime/crt/packed_func.h rename to src/runtime/crt/common/packed_func.c index d4597e62fd0f..81dfcb54c303 100644 --- a/src/runtime/crt/packed_func.h +++ b/src/runtime/crt/common/packed_func.c @@ -17,21 +17,17 @@ * under the License. */ +// LINT_C_FILE + /*! - * \file tvm/runtime/packed_func.h - * \brief Type-erased function used across TVM API. + * \file src/runtime/crt/common/packed_func.c + * \brief PackedFunc implementation. */ -#ifndef TVM_RUNTIME_CRT_PACKED_FUNC_H_ -#define TVM_RUNTIME_CRT_PACKED_FUNC_H_ - -#include -#include -#include -#include - -#include "module.h" +#include +#include +#include -static inline DLDataType String2DLDataType(const char* s) { +DLDataType String2DLDataType(const char* s) { DLDataType t; // handle None type if (strlen(s) == 0) { @@ -78,13 +74,38 @@ static inline DLDataType String2DLDataType(const char* s) { return t; } -typedef struct TVMArgs { - TVMValue values[TVM_CRT_MAX_ARGS]; - int tcodes[TVM_CRT_MAX_ARGS]; /* Data type should be identical to type_codes in TVMPackedCFunc */ - uint32_t values_count; -} TVMArgs; +int TVMPackedFunc_InitGlobalFunc(TVMPackedFunc* pf, const char* name, const TVMArgs* args) { + int status = 0; + + pf->Call = &TVMPackedFunc_Call; + pf->SetArgs = &TVMPackedFunc_SetArgs; + + status = TVMFuncGetGlobal(name, &pf->fexec); + if (status != 0) { + return status; + } + + TVMPackedFunc_SetArgs(pf, args); + return status; +} + +int TVMPackedFunc_InitModuleFunc(TVMPackedFunc* pf, TVMModuleHandle module, const char* name, + const TVMArgs* args) { + int status = 0; + + pf->Call = &TVMPackedFunc_Call; + pf->SetArgs = &TVMPackedFunc_SetArgs; + + status = TVMModGetFunction(module, name, 0, &pf->fexec); + if (status != 0) { + return status; + } -static inline TVMArgs TVMArgs_Create(TVMValue* values, uint32_t* tcodes, uint32_t values_count) { + TVMPackedFunc_SetArgs(pf, args); + return status; +} + +TVMArgs TVMArgs_Create(TVMValue* values, uint32_t* tcodes, uint32_t values_count) { uint32_t idx; TVMArgs args; memset(&args, 0, sizeof(args)); @@ -96,49 +117,14 @@ static inline TVMArgs TVMArgs_Create(TVMValue* values, uint32_t* tcodes, uint32_ return args; } -static inline int TVMNoOperation(TVMValue* args, int* type_codes, int num_args, - TVMRetValueHandle ret, void* res) { - return 0; -} - -typedef struct TVMPackedFunc { - char name[200]; - TVMPackedCFunc fexec; - TVMArgs args; - void (*Call)(struct TVMPackedFunc* pf); - void (*SetArgs)(struct TVMPackedFunc* pf, const struct TVMArgs* args); -} TVMPackedFunc; - -static inline void TVMPackedFunc_Call(TVMPackedFunc* pf) { - pf->fexec(pf->args.values, pf->args.tcodes, pf->args.values_count, 0, 0); +int TVMPackedFunc_Call(TVMPackedFunc* pf) { + return TVMFuncCall(pf->fexec, pf->args.values, pf->args.tcodes, pf->args.values_count, + pf->ret_value.values, pf->ret_value.tcodes); } -static inline void TVMPackedFunc_SetArgs(TVMPackedFunc* pf, const TVMArgs* args) { +void TVMPackedFunc_SetArgs(TVMPackedFunc* pf, const TVMArgs* args) { memcpy(&(pf->args), args, sizeof(TVMArgs)); } -TVMPackedFunc* g_fexecs = 0; -uint32_t g_fexecs_count = 0; - -// Implement TVMModule::GetFunction -// Put implementation in this file so we have seen the TVMPackedFunc -static inline void TVMModule_GetFunction(TVMModule* mod, const char* name, TVMPackedFunc* pf) { - int idx; - memset(pf, 0, sizeof(TVMPackedFunc)); - assert(strlen(name) <= sizeof(pf->name)); - snprintf(pf->name, strlen(name), "%s", name); - pf->Call = TVMPackedFunc_Call; - pf->SetArgs = TVMPackedFunc_SetArgs; - pf->fexec = &TVMNoOperation; - for (idx = 0; idx < g_fexecs_count; idx++) { - if (!strcmp(g_fexecs[idx].name, name)) { - pf->fexec = g_fexecs[idx].fexec; - break; - } - } - if (idx == g_fexecs_count) { - fprintf(stderr, "function handle for %s not found\n", name); - } -} - -#endif // TVM_RUNTIME_CRT_PACKED_FUNC_H_ +TVMPackedFunc* g_fexecs; +uint32_t g_fexecs_count; diff --git a/src/runtime/crt/crt_runtime_api.c b/src/runtime/crt/crt_runtime_api.c deleted file mode 100644 index bd7d35e119bc..000000000000 --- a/src/runtime/crt/crt_runtime_api.c +++ /dev/null @@ -1,97 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -#include -#include -#include -#include -#include - -#include "graph_runtime.h" -#include "ndarray.h" -#include "packed_func.h" - -// Handle internal errors - -static char g_last_error[1024]; - -void TVMAPISetLastError(const char* msg) { - assert(strlen(msg) < sizeof(g_last_error)); - snprintf(g_last_error, sizeof(g_last_error), "%s", msg); -} - -const char* TVMGetLastError(void) { return g_last_error; } - -// Manipulate NDArray on target device - -int TVMArrayAlloc(const tvm_index_t* shape, int ndim, int dtype_code, int dtype_bits, - int dtype_lanes, int device_type, int device_id, TVMArrayHandle* out) { - DLDataType dtype; - dtype.code = dtype_code; - dtype.bits = dtype_bits; - dtype.lanes = dtype_lanes; - DLContext ctx; - ctx.device_type = (DLDeviceType)device_type; - ctx.device_id = device_id; - TVMNDArray arr = TVMNDArray_Empty(ndim, shape, dtype, ctx); - **out = arr.dl_tensor; - return 0; -} - -int TVMArrayFree(TVMArrayHandle handle) { - TVMNDArray arr; - arr.dl_tensor = *handle; - return TVMNDArray_Release(&arr); -} - -void* SystemLibraryCreate() { return 0; } - -int TVMModGetFunction(TVMModuleHandle mod, const char* func_name, int query_imports, - TVMFunctionHandle* out) { - int status = 0; - if (!strcmp(func_name, "load_params")) { - *out = &TVMGraphRuntime_LoadParams; - } else { - status = -1; - } - return status; -} - -int TVMFuncGetGlobal(const char* name, TVMFunctionHandle* out) { - int status = 0; - if (!strcmp(name, "tvm.graph_runtime.create")) { - *out = &TVMGraphRuntimeCreate; - } else if (!strcmp(name, "tvm.graph_runtime.set_input")) { - *out = &TVMGraphRuntime_SetInput; - } else if (!strcmp(name, "tvm.graph_runtime.run")) { - *out = &TVMGraphRuntime_Run; - } else if (!strcmp(name, "tvm.graph_runtime.get_output")) { - *out = &TVMGraphRuntime_GetOutput; - } else if (!strcmp(name, "tvm.graph_runtime.release")) { - *out = &TVMGraphRuntimeRelease; - } else if (!strcmp(name, "runtime.SystemLib")) { - *out = &SystemLibraryCreate; - } else { - char msg[200]; - snprintf(msg, sizeof(msg), "fail to get global: name=%s", name); - TVMAPISetLastError(msg); - status = -1; - } - return status; -} diff --git a/src/runtime/crt/graph_runtime.h b/src/runtime/crt/graph_runtime.h deleted file mode 100644 index fd3b14633222..000000000000 --- a/src/runtime/crt/graph_runtime.h +++ /dev/null @@ -1,215 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -/*! - * \file graph_runtime.h - * \brief Tiny graph runtime that can run graph containing only tvm PackedFunc. - */ -#ifndef TVM_RUNTIME_CRT_GRAPH_RUNTIME_H_ -#define TVM_RUNTIME_CRT_GRAPH_RUNTIME_H_ - -#include - -#include "load_json.h" -#include "module.h" -#include "ndarray.h" -#include "packed_func.h" - -/*! \brief operator attributes about tvm op */ -typedef struct TVMOpParam { - char func_name[120]; - uint32_t num_inputs; - uint32_t num_outputs; - uint32_t flatten_data; -} TVMOpParam; - -// Memory pool entry. -typedef struct TVMGraphRuntimePoolEntry { - size_t size; - int device_type; -} TVMGraphRuntimePoolEntry; - -// Node entry -typedef struct TVMGraphRuntimeNodeEntry { - uint32_t node_id; - uint32_t index; - uint32_t version; - // JSON Loader - void (*Load)(JSONReader* reader); -} TVMGraphRuntimeNodeEntry; - -// Node -typedef struct TVMGraphRuntimeNode { - // operator type in string - char op_type[16]; - // name of the op - char name[120]; - // parameters - TVMOpParam param; - // inputs - TVMGraphRuntimeNodeEntry* inputs; - // number of inputs - size_t inputs_count; - // control deps - uint32_t control_deps[20]; - // JSON Loader - void (*LoadAttrs)(struct TVMGraphRuntimeNode* node, JSONReader* reader, TVMOpParam* param); - // JSON Loader - int (*Load)(struct TVMGraphRuntimeNode* node, JSONReader* reader); -} TVMGraphRuntimeNode; - -// Graph attribute -typedef struct TVMGraphRuntimeGraphAttr { - uint32_t storage_num_not_alloctaed; - uint32_t* storage_id; - uint32_t* device_index; - char* dltype; // "int8", "int16", "float32" - uint32_t dltype_count; - int64_t* shape; - uint32_t* ndim; - uint32_t shape_count; -} TVMGraphRuntimeGraphAttr; - -typedef DLTensor* DLTensorPtr; - -/*! - * \brief Tiny graph runtime. - * - * This runtime can be acccesibly in various language via - * TVM runtime PackedFunc API. - */ -/* class GraphRuntime : public ModuleNode { */ -typedef struct TVMGraphRuntime { - void (*Run)(struct TVMGraphRuntime* runtime); - - /*! - * \brief Initialize the graph executor with graph and context. - * \param runtime The graph runtime. - * \param graph_json The execution graph. - * \param module The module containing the compiled functions for the host - * processor. - * \param ctxs The context of the host and devices where graph nodes will be - * executed on. - */ - void (*Init)(struct TVMGraphRuntime* runtime, const char* graph_json, const TVMModule* module, - const TVMContext* ctxs); - - /*! - * \brief Get the input index given the name of input. - * \param runtime The graph runtime. - * \param name The name of the input. - * \return The index of input. - */ - int (*GetInputIndex)(struct TVMGraphRuntime* runtime, const char* name); - - /*! - * \brief set input to the graph based on name. - * \param runtime The graph runtime. - * \param name The name of the input. - * \param data_in The input data. - */ - void (*SetInput)(struct TVMGraphRuntime* runtime, const char* name, DLTensor* data_in); - - /*! - * \brief Return NDArray for given output index. - * \param runtime The graph runtime. - * \param index The output index. - * \param out The DLTensor corresponding to given output node index. - * \return The result of this function execution. - */ - int (*GetOutput)(struct TVMGraphRuntime* runtime, const int32_t index, DLTensor* out); - /*! - * \brief Load parameters from parameter blob. - * \param runtime The graph runtime. - * \param param_blob A binary blob of parameter. - * \param param_size The parameter size. - * \return The result of this function execution. - */ - int (*LoadParams)(struct TVMGraphRuntime* runtime, const char* param_blob, - const uint32_t param_size); - - // The graph attribute fields. - int (*Load)(struct TVMGraphRuntime* runtime, JSONReader* reader); - /*! \brief Setup the temporal storage */ - void (*SetupStorage)(struct TVMGraphRuntime* runtime); - /*! \brief Setup the executors. */ - int (*SetupOpExecs)(struct TVMGraphRuntime* runtime); - - /*! - * \brief Create an execution function given input. - * \param runtime The graph runtime. - * \param attrs The node attributes. - * \param args The arguments to the functor, including inputs and outputs. - * \param args_count The total number of arguments. - * \param num_inputs Number of inputs. - * \param pf The created executor. - * \return The result of this function execution. - */ - int32_t (*CreateTVMOp)(struct TVMGraphRuntime* runtime, const TVMOpParam* attrs, - DLTensorPtr* args, const uint32_t args_count, uint32_t num_inputs, - TVMPackedFunc* pf); - - // Get node entry index. - uint32_t (*GetEntryId)(struct TVMGraphRuntime* runtime, uint32_t nid, uint32_t index); - - /*! \brief The graph nodes. */ - TVMGraphRuntimeNode* nodes; - /*! \brief The graph nodes counter. */ - uint32_t nodes_count; - /*! \brief The argument nodes. */ - uint32_t* input_nodes; - uint32_t input_nodes_count; - /*! \brief Used for quick entry indexing. */ - uint32_t* node_row_ptr; - uint32_t node_row_ptr_count; - /*! \brief Output entries. */ - TVMGraphRuntimeNodeEntry* outputs; - /*! \brief Output entries counter. */ - uint32_t outputs_count; - /*! \brief Additional graph attributes. */ - TVMGraphRuntimeGraphAttr attrs; - /*! \brief The code module that contains both host and device code. */ - TVMModule module; - /*! \brief Execution context of all devices including the host. */ - TVMContext ctxs[1]; - uint32_t ctxs_count; - /*! \brief Common storage pool for all devices. */ - TVMNDArray* storage_pool; - uint32_t storage_pool_count; - /*! \brief Data entry of each node. */ - TVMNDArray* data_entry; - uint32_t data_entry_count; - /*! \brief Operator on each node. */ - TVMPackedFunc* op_execs; - uint32_t op_execs_count; -} TVMGraphRuntime; - -// public functions -TVMGraphRuntime* TVMGraphRuntimeCreate(const char* sym_json, const TVMModule* m, - const TVMContext* ctxs); -void TVMGraphRuntimeRelease(TVMGraphRuntime** runtime); - -// private functions -void TVMGraphRuntime_SetInput(TVMGraphRuntime* runtime, const char* name, DLTensor* data_in); -int TVMGraphRuntime_LoadParams(TVMGraphRuntime* runtime, const char* param_blob, - const uint32_t param_size); -void TVMGraphRuntime_Run(TVMGraphRuntime* runtime); -int TVMGraphRuntime_GetOutput(TVMGraphRuntime* runtime, const int32_t idx, DLTensor* out); - -#endif // TVM_RUNTIME_CRT_GRAPH_RUNTIME_H_ diff --git a/src/runtime/crt/graph_runtime.c b/src/runtime/crt/graph_runtime/graph_runtime.c similarity index 93% rename from src/runtime/crt/graph_runtime.c rename to src/runtime/crt/graph_runtime/graph_runtime.c index 0ddbb41ae730..cf56a5c8b773 100644 --- a/src/runtime/crt/graph_runtime.c +++ b/src/runtime/crt/graph_runtime/graph_runtime.c @@ -17,16 +17,21 @@ * under the License. */ +// LINT_C_FILE + /*! * \file graph_runtime.c * \brief implement graph runtime in pure C */ -#include "graph_runtime.h" - +#include +#include +#include #include +#include +#include -#include "logging.h" +#include "crt_config.h" #ifndef MAX #define MAX(a, b) (((a) > (b)) ? (a) : (b)) @@ -529,11 +534,11 @@ int TVMGraphRuntime_GetInputIndex(TVMGraphRuntime* runtime, const char* name) { * \param data_in The input data. */ void TVMGraphRuntime_SetInput(TVMGraphRuntime* runtime, const char* name, DLTensor* data_in) { - uint32_t index = runtime->GetInputIndex(runtime, name); + uint32_t index = TVMGraphRuntime_GetInputIndex(runtime, name); if (index >= runtime->input_nodes_count) { fprintf(stderr, "given index is greater than num of input nodes.\n"); } - uint32_t eid = runtime->GetEntryId(runtime, runtime->input_nodes[index], 0); + uint32_t eid = TVMGraphRuntime_GetEntryId(runtime, runtime->input_nodes[index], 0); runtime->data_entry[eid].dl_tensor.data = data_in->data; } @@ -588,10 +593,10 @@ int TVMGraphRuntime_LoadParams(TVMGraphRuntime* runtime, const char* param_blob, } for (idx = 0; idx < size; idx++) { - int32_t in_idx = runtime->GetInputIndex(runtime, names + TVM_CRT_STRLEN_NAME * idx); + int32_t in_idx = TVMGraphRuntime_GetInputIndex(runtime, names + TVM_CRT_STRLEN_NAME * idx); CHECK_GT(in_idx, 0, "Found param for non-existent input: %s\n", names + TVM_CRT_STRLEN_NAME * idx); - uint32_t eid = runtime->GetEntryId(runtime, runtime->input_nodes[in_idx], 0); + uint32_t eid = TVMGraphRuntime_GetEntryId(runtime, runtime->input_nodes[in_idx], 0); if (!(eid < runtime->data_entry_count)) { fprintf(stderr, "`entry_id`=%d is greater than expected(%d).\n", eid, runtime->data_entry_count); @@ -642,7 +647,7 @@ int TVMGraphRuntime_GetOutput(TVMGraphRuntime* runtime, const int32_t idx, DLTen int status = 0; uint32_t nid = runtime->outputs[idx].node_id; uint32_t index = runtime->outputs[idx].index; - uint32_t eid = runtime->GetEntryId(runtime, nid, index); + uint32_t eid = TVMGraphRuntime_GetEntryId(runtime, nid, index); // copy data section to allocated output tensor int32_t elem_bytes = out->dtype.bits / 8; @@ -737,12 +742,12 @@ int TVMGraphRuntime_SetupOpExecs(TVMGraphRuntime* runtime) { uint32_t args_count = 0; for (idx = 0; idx < inode->inputs_count; idx++) { const TVMGraphRuntimeNodeEntry* entry = inode->inputs + idx; - uint32_t eid = runtime->GetEntryId(runtime, entry->node_id, entry->index); + uint32_t eid = TVMGraphRuntime_GetEntryId(runtime, entry->node_id, entry->index); args[idx] = &(runtime->data_entry[eid].dl_tensor); args_count++; } for (idx = 0; idx < inode->param.num_outputs; idx++) { - uint32_t eid = runtime->GetEntryId(runtime, nid, idx); + uint32_t eid = TVMGraphRuntime_GetEntryId(runtime, nid, idx); args[args_count] = &(runtime->data_entry[eid].dl_tensor); args_count++; } @@ -761,7 +766,8 @@ int TVMGraphRuntime_SetupOpExecs(TVMGraphRuntime* runtime) { printf("tvm_op: creating %s with node_id=%d\n", inode->param.func_name, nid); #endif // TVM_CRT_DEBUG TVMPackedFunc pf; - runtime->CreateTVMOp(runtime, &(inode->param), args, args_count, inode->inputs_count, &pf); + TVMGraphRuntime_CreateTVMOp(runtime, &(inode->param), args, args_count, inode->inputs_count, + &pf); runtime->op_execs[nid] = pf; } } @@ -811,9 +817,8 @@ int32_t TVMGraphRuntime_CreateTVMOp(TVMGraphRuntime* runtime, const TVMOpParam* status = -1; } - runtime->module.GetFunction(&(runtime->module), param->func_name, pf); TVMArgs targs = TVMArgs_Create(arg_ptr.arg_values, arg_ptr.arg_tcodes, arg_ptr.arg_values_count); - pf->SetArgs(pf, &targs); + status = TVMPackedFunc_InitModuleFunc(pf, runtime->module_handle, param->func_name, &targs); return status; } @@ -829,37 +834,26 @@ int32_t TVMGraphRuntime_CreateTVMOp(TVMGraphRuntime* runtime, const TVMOpParam* void TVMGraphRuntime_Init(TVMGraphRuntime* runtime, const char* graph_json, const TVMModule* module, const TVMContext* ctxs) { JSONReader reader = JSONReader_Create(graph_json); - runtime->Load(runtime, &reader); + TVMGraphRuntime_Load(runtime, &reader); JSONReader_Release(&reader); runtime->ctxs[0] = ctxs[0]; - runtime->SetupStorage(runtime); - runtime->SetupOpExecs(runtime); + TVMGraphRuntime_SetupStorage(runtime); + TVMGraphRuntime_SetupOpExecs(runtime); } -TVMGraphRuntime* TVMGraphRuntimeCreate(const char* sym_json, const TVMModule* m, - const TVMContext* ctxs) { +TVMGraphRuntime* TVMGraphRuntime_Create(const char* sym_json, const TVMModule* m, + const TVMContext* ctxs) { + CHECK_EQ(vleak_size, 1, "memory leak checking won't work with concurrent CRT use"); TVMGraphRuntime* runtime = (TVMGraphRuntime*)vmalloc(sizeof(TVMGraphRuntime)); // NOLINT(*) memset(runtime, 0, sizeof(TVMGraphRuntime)); - runtime->GetEntryId = TVMGraphRuntime_GetEntryId; - runtime->GetInputIndex = TVMGraphRuntime_GetInputIndex; - runtime->Init = TVMGraphRuntime_Init; - runtime->Load = TVMGraphRuntime_Load; - runtime->SetInput = TVMGraphRuntime_SetInput; - runtime->LoadParams = TVMGraphRuntime_LoadParams; - runtime->Run = TVMGraphRuntime_Run; - runtime->GetOutput = TVMGraphRuntime_GetOutput; - runtime->SetupStorage = TVMGraphRuntime_SetupStorage; - runtime->SetupOpExecs = TVMGraphRuntime_SetupOpExecs; - runtime->CreateTVMOp = TVMGraphRuntime_CreateTVMOp; - runtime->module.GetFunction = TVMModule_GetFunction; // init - runtime->Init(runtime, sym_json, m, ctxs); + TVMGraphRuntime_Init(runtime, sym_json, m, ctxs); return runtime; } -void TVMGraphRuntimeRelease(TVMGraphRuntime** pptr) { +void TVMGraphRuntime_Release(TVMGraphRuntime** pptr) { int32_t idx; - TVMGraphRuntime* runtime = *pptr; + TVMGraphRuntime* runtime = (TVMGraphRuntime*)(*pptr); for (idx = 0; idx < runtime->nodes_count; ++idx) { TVMGraphRuntimeNodeRelease(&(runtime->nodes[idx])); } @@ -884,5 +878,5 @@ void TVMGraphRuntimeRelease(TVMGraphRuntime** pptr) { g_fexecs = 0; } - CHECK_EQ(vleak_size, 0, "found memory leak, leak size=%d", vleak_size); + CHECK_EQ(vleak_size, 1, "found memory leak, leak size=%d", vleak_size - 1); } diff --git a/src/runtime/crt/load_json.c b/src/runtime/crt/graph_runtime/load_json.c similarity index 98% rename from src/runtime/crt/load_json.c rename to src/runtime/crt/graph_runtime/load_json.c index 5ae60cca86b6..e4c71fd55095 100644 --- a/src/runtime/crt/load_json.c +++ b/src/runtime/crt/graph_runtime/load_json.c @@ -17,12 +17,15 @@ * under the License. */ +// LINT_C_FILE + /*! * \file load_json.c * \brief Load graph from JSON file. */ -#include "load_json.h" - +#include +#include +#include #include // the node entry structure in serialized format diff --git a/src/runtime/crt/host/crt_config.h b/src/runtime/crt/host/crt_config.h new file mode 100644 index 000000000000..c0b02a69ba5b --- /dev/null +++ b/src/runtime/crt/host/crt_config.h @@ -0,0 +1,68 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +/*! + * \file tvm/runtime/crt/host/crt_config.h + * \brief CRT configuration for the host-linked CRT. + */ +#ifndef TVM_RUNTIME_CRT_HOST_CRT_CONFIG_H_ +#define TVM_RUNTIME_CRT_HOST_CRT_CONFIG_H_ + +/*! Support low-level debugging in MISRA-C runtime */ +#define TVM_CRT_DEBUG 0 + +/*! Maximum supported dimension in NDArray */ +#define TVM_CRT_MAX_NDIM 6 +/*! Maximum supported arguments in generated functions */ +#define TVM_CRT_MAX_ARGS 10 +/*! Maximum supported string length in dltype, e.g. "int8", "int16", "float32" */ +#define TVM_CRT_STRLEN_DLTYPE 10 +/*! Maximum supported string length in function names */ +#define TVM_CRT_STRLEN_NAME 80 + +/*! + * \brief Log memory pool size for virtual memory allocation + * + * Here is a list of possible choices: + * * use 16 for 64 KiB memory space + * * use 17 for 128 KiB memory space + * * use 18 for 256 KiB memory space + * * use 19 for 512 KiB memory space + * * use 20 for 1 MiB memory space + * * use 21 for 2 MiB memory space + * * use 22 for 4 MiB memory space + * * use 23 for 8 MiB memory space + * * use 24 for 16 MiB memory space + * * use 25 for 32 MiB memory space + * * use 26 for 64 MiB memory space + * * use 27 for 128 MiB memory space + * * use 28 for 256 MiB memory space + */ +#define TVM_CRT_LOG_VIRT_MEM_SIZE 24 + +/*! \brief Log2 of page size for virtual memory allocation */ +#define TVM_CRT_PAGE_BYTES_LOG 12 + +/*! Maximum number of registered modules. */ +#define TVM_CRT_MAX_REGISTERED_MODULES 2 + +/*! Size of the global function registry, in bytes. */ +#define TVM_CRT_GLOBAL_FUNC_REGISTRY_SIZE_BYTES 200 + +#endif // TVM_RUNTIME_CRT_HOST_CRT_CONFIG_H_ diff --git a/src/runtime/crt/include/tvm/runtime/crt/internal/common/func_registry.h b/src/runtime/crt/include/tvm/runtime/crt/internal/common/func_registry.h new file mode 100644 index 000000000000..d62e3d7e1104 --- /dev/null +++ b/src/runtime/crt/include/tvm/runtime/crt/internal/common/func_registry.h @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +// LINT_C_FILE + +/*! + * \file tvm/runtime/crt/include/tvm/runtime/crt/internal/common/func_registry.h + * \brief Abstract device memory management API + */ +#ifndef TVM_RUNTIME_CRT_INCLUDE_TVM_RUNTIME_CRT_INTERNAL_COMMON_FUNC_REGISTRY_H_ +#define TVM_RUNTIME_CRT_INCLUDE_TVM_RUNTIME_CRT_INTERNAL_COMMON_FUNC_REGISTRY_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +int strcmp_cursor(const char** cursor, const char* name); + +#ifdef __cplusplus +} // extern "C" +#endif + +#endif // TVM_RUNTIME_CRT_INCLUDE_TVM_RUNTIME_CRT_INTERNAL_COMMON_FUNC_REGISTRY_H_ diff --git a/src/runtime/crt/logging.h b/src/runtime/crt/include/tvm/runtime/crt/internal/common/logging.h similarity index 89% rename from src/runtime/crt/logging.h rename to src/runtime/crt/include/tvm/runtime/crt/internal/common/logging.h index c711b3aa3bb9..17fbe32a1f2c 100644 --- a/src/runtime/crt/logging.h +++ b/src/runtime/crt/include/tvm/runtime/crt/internal/common/logging.h @@ -18,13 +18,13 @@ */ /*! - * \file runtime/crt/loggin.h + * \file runtime/crt/include/tvm/runtime/crt/internal/common/logging.h * \brief A replacement of the dmlc logging system that avoids * the usage of GLOG and C++ headers */ -#ifndef TVM_RUNTIME_CRT_LOGGING_H_ -#define TVM_RUNTIME_CRT_LOGGING_H_ +#ifndef TVM_RUNTIME_CRT_INCLUDE_TVM_RUNTIME_CRT_INTERNAL_COMMON_LOGGING_H_ +#define TVM_RUNTIME_CRT_INCLUDE_TVM_RUNTIME_CRT_INTERNAL_COMMON_LOGGING_H_ #ifndef CHECK #define CHECK(x) \ @@ -70,4 +70,4 @@ #define CHECK_NE(x, y, fmt, ...) CHECK_BINARY_OP(!=, x, y, fmt, ##__VA_ARGS__) #endif -#endif // TVM_RUNTIME_CRT_LOGGING_H_ +#endif // TVM_RUNTIME_CRT_INCLUDE_TVM_RUNTIME_CRT_INTERNAL_COMMON_LOGGING_H_ diff --git a/src/runtime/crt/include/tvm/runtime/crt/internal/common/memory.h b/src/runtime/crt/include/tvm/runtime/crt/internal/common/memory.h new file mode 100644 index 000000000000..8162fd7851a2 --- /dev/null +++ b/src/runtime/crt/include/tvm/runtime/crt/internal/common/memory.h @@ -0,0 +1,141 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +/*! + * \file runtime/crt/include/tvm/runtime/crt/internal/common/memory.h + * \brief Defines data types and functions used in the internal memory manager. + * Exposed for testing. + */ + +#ifndef TVM_RUNTIME_CRT_INCLUDE_TVM_RUNTIME_CRT_INTERNAL_COMMON_MEMORY_H_ +#define TVM_RUNTIME_CRT_INCLUDE_TVM_RUNTIME_CRT_INTERNAL_COMMON_MEMORY_H_ + +#include + +#include "crt_config.h" + +#ifdef __cplusplus +extern "C" { +#endif + +/*! Number of bits in a page */ +#define TVM_CRT_PAGE_BITS ((1 << TVM_CRT_PAGE_BYTES_LOG) << 3) + +/*! \brief Translate log memory size into bytes */ +#define TVM_CRT_VIRT_MEM_SIZE (1 << TVM_CRT_LOG_VIRT_MEM_SIZE) + +/*! \brief Number of possible page entries in total */ +#define TVM_CRT_MAX_PAGES (TVM_CRT_VIRT_MEM_SIZE / TVM_CRT_PAGE_BYTES) + +/*! \brief A page in the DRAM */ +typedef struct Page { + /*! \brief Start location in page table */ + tvm_index_t ptable_begin; + /*! \brief The total number of pages */ + tvm_index_t num_pages; + /*! \brief Data */ + uint8_t* data; +} Page; + +// construct a new page +Page PageCreate(uint8_t* memory_pool, size_t page_size_bytes, tvm_index_t ptable_begin, + tvm_index_t num_pages); + +typedef struct PageTable { + // Pointer to beginning of memory pool. + uint8_t* memory_pool; + // Size of one page. + size_t page_size_bytes; + + Page* page; + size_t max_pages; + size_t num_pages; + void (*resize)(struct PageTable* ptable, size_t size, Page* page); +} PageTable; + +typedef struct PageEntry { + uint8_t* addr; + Page page; +} PageEntry; + +typedef struct TLB { + PageEntry* entries; + size_t max_pages; + uint32_t num_pages; + void (*set)(struct TLB* tlb, uint8_t* data, Page* page); + PageEntry* (*find)(struct TLB* tlb, uint8_t* data); +} TLB; + +typedef struct IndexedEntry { + tvm_index_t index; + Page page; +} IndexedEntry; + +typedef struct MultiMap { + IndexedEntry* entries; + size_t max_entries; + size_t num_entries; + IndexedEntry* (*lower_bound)(struct MultiMap* map, uint32_t npage); + IndexedEntry* (*end)(struct MultiMap* map); + void (*erase)(struct MultiMap* map, IndexedEntry* entry); + void (*insert)(struct MultiMap* map, uint32_t npage, Page* p); +} MultiMap; + +/*! + * \brief DRAM memory manager + * Implements simple paging to allow physical address translation. + */ +typedef struct MemoryManager { + /*! + * \brief Allocate memory from manager + * \param size The size of memory + * \return The virtual address + */ + void* (*Alloc)(struct MemoryManager* mgr, tvm_index_t size); + /*! + * \brief Allocate memory from manager + * \param ptr The pointer to the memory area to be reallocated + * \param size The size of memory + * \return The virtual address + */ + void* (*Realloc)(struct MemoryManager* mgr, void* ptr, tvm_index_t size); + /*! + * \brief Free the memory. + * \param ptr The pointer to the memory to deallocate + * \return The virtual address + */ + void (*Free)(struct MemoryManager* mgr, void* data); + + // Physical address -> page + PageTable ptable; + // Virtual address -> page + TLB pmap; + // Free map + MultiMap free_map; +} MemoryManager; + +// Exposed for testing +void MemoryManagerCreate(MemoryManager* manager, uint8_t* memory_pool, + size_t memory_pool_size_bytes, size_t page_size_bytes_log2); + +#ifdef __cplusplus +} // extern "C" +#endif + +#endif // TVM_RUNTIME_CRT_INCLUDE_TVM_RUNTIME_CRT_INTERNAL_COMMON_MEMORY_H_ diff --git a/src/runtime/crt/ndarray.h b/src/runtime/crt/include/tvm/runtime/crt/internal/common/ndarray.h similarity index 85% rename from src/runtime/crt/ndarray.h rename to src/runtime/crt/include/tvm/runtime/crt/internal/common/ndarray.h index ae76726ae0b9..8da4b3c720c0 100644 --- a/src/runtime/crt/ndarray.h +++ b/src/runtime/crt/include/tvm/runtime/crt/internal/common/ndarray.h @@ -18,11 +18,11 @@ */ /*! - * \file tvm/runtime/crt/ndarray.h + * \file tvm/runtime/crt/include/tvm/runtime/crt/internal/common/ndarray.h * \brief Abstract device memory management API */ -#ifndef TVM_RUNTIME_CRT_NDARRAY_H_ -#define TVM_RUNTIME_CRT_NDARRAY_H_ +#ifndef TVM_RUNTIME_CRT_INCLUDE_TVM_RUNTIME_CRT_INTERNAL_COMMON_NDARRAY_H_ +#define TVM_RUNTIME_CRT_INCLUDE_TVM_RUNTIME_CRT_INTERNAL_COMMON_NDARRAY_H_ #include #include @@ -54,4 +54,4 @@ TVMNDArray TVMNDArray_CreateView(TVMNDArray* arr, const tvm_index_t* shape, uint int TVMNDArray_Release(TVMNDArray* arr); -#endif // TVM_RUNTIME_CRT_NDARRAY_H_ +#endif // TVM_RUNTIME_CRT_INCLUDE_TVM_RUNTIME_CRT_INTERNAL_COMMON_NDARRAY_H_ diff --git a/src/runtime/crt/include/tvm/runtime/crt/internal/graph_runtime/graph_runtime.h b/src/runtime/crt/include/tvm/runtime/crt/internal/graph_runtime/graph_runtime.h new file mode 100644 index 000000000000..7ea7a4f035c8 --- /dev/null +++ b/src/runtime/crt/include/tvm/runtime/crt/internal/graph_runtime/graph_runtime.h @@ -0,0 +1,113 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +/*! + * \file src/runtime/crt/include/tvm/runtime/crt/internal/graph_runtime/graph_runtime.h + * \brief Tiny graph runtime that can run graph containing only tvm PackedFunc. + */ +#ifndef TVM_RUNTIME_CRT_INCLUDE_TVM_RUNTIME_CRT_INTERNAL_GRAPH_RUNTIME_GRAPH_RUNTIME_H_ +#define TVM_RUNTIME_CRT_INCLUDE_TVM_RUNTIME_CRT_INTERNAL_GRAPH_RUNTIME_GRAPH_RUNTIME_H_ + +#include +#include +#include +#include + +// Memory pool entry. +typedef struct TVMGraphRuntimePoolEntry { + size_t size; + int device_type; +} TVMGraphRuntimePoolEntry; + +// Node entry +typedef struct TVMGraphRuntimeNodeEntry { + uint32_t node_id; + uint32_t index; + uint32_t version; + // JSON Loader + void (*Load)(JSONReader* reader); +} TVMGraphRuntimeNodeEntry; + +// Node +typedef struct TVMGraphRuntimeNode { + // operator type in string + char op_type[16]; + // name of the op + char name[120]; + // parameters + TVMOpParam param; + // inputs + TVMGraphRuntimeNodeEntry* inputs; + // number of inputs + size_t inputs_count; + // control deps + uint32_t control_deps[20]; + // JSON Loader + void (*LoadAttrs)(struct TVMGraphRuntimeNode* node, JSONReader* reader, TVMOpParam* param); + // JSON Loader + int (*Load)(struct TVMGraphRuntimeNode* node, JSONReader* reader); +} TVMGraphRuntimeNode; + +typedef struct TVMGraphRuntime { + /*! \brief The graph nodes. */ + TVMGraphRuntimeNode* nodes; + /*! \brief The graph nodes counter. */ + uint32_t nodes_count; + /*! \brief The argument nodes. */ + uint32_t* input_nodes; + uint32_t input_nodes_count; + /*! \brief Used for quick entry indexing. */ + uint32_t* node_row_ptr; + uint32_t node_row_ptr_count; + /*! \brief Output entries. */ + TVMGraphRuntimeNodeEntry* outputs; + /*! \brief Output entries counter. */ + uint32_t outputs_count; + /*! \brief Additional graph attributes. */ + TVMGraphRuntimeGraphAttr attrs; + /*! \brief The code module that contains both host and device code. */ + TVMModuleHandle module_handle; + /*! \brief Execution context of all devices including the host. */ + TVMContext ctxs[1]; + uint32_t ctxs_count; + /*! \brief Common storage pool for all devices. */ + TVMNDArray* storage_pool; + uint32_t storage_pool_count; + /*! \brief Data entry of each node. */ + TVMNDArray* data_entry; + uint32_t data_entry_count; + /*! \brief Operator on each node. */ + TVMPackedFunc* op_execs; + uint32_t op_execs_count; +} TVMGraphRuntime; + +typedef DLTensor* DLTensorPtr; + +// private functions +void TVMGraphRuntime_SetInput(TVMGraphRuntime* runtime, const char* name, DLTensor* data_in); +int TVMGraphRuntime_LoadParams(TVMGraphRuntime* runtime, const char* param_blob, + const uint32_t param_size); +void TVMGraphRuntime_Run(TVMGraphRuntime* runtime); +int TVMGraphRuntime_GetOutput(TVMGraphRuntime* runtime, const int32_t idx, DLTensor* out); + +int32_t TVMGraphRuntime_CreateTVMOp(TVMGraphRuntime* runtime, const TVMOpParam* param, + DLTensorPtr* args, const uint32_t args_count, + uint32_t num_inputs, TVMPackedFunc* pf); + +#endif // TVM_RUNTIME_CRT_INCLUDE_TVM_RUNTIME_CRT_INTERNAL_GRAPH_RUNTIME_GRAPH_RUNTIME_H_ diff --git a/src/runtime/crt/load_json.h b/src/runtime/crt/include/tvm/runtime/crt/internal/graph_runtime/load_json.h similarity index 88% rename from src/runtime/crt/load_json.h rename to src/runtime/crt/include/tvm/runtime/crt/internal/graph_runtime/load_json.h index 0c9324777c1d..39c2576b2282 100644 --- a/src/runtime/crt/load_json.h +++ b/src/runtime/crt/include/tvm/runtime/crt/internal/graph_runtime/load_json.h @@ -18,13 +18,14 @@ */ /*! - * \file load_json.h + * \file src/runtime/crt/include/tvm/runtime/crt/internal/graph_runtime/load_json.h * \brief Lightweight JSON Reader that read save into C++ data structs. */ -#ifndef TVM_RUNTIME_CRT_LOAD_JSON_H_ -#define TVM_RUNTIME_CRT_LOAD_JSON_H_ +#ifndef TVM_RUNTIME_CRT_INCLUDE_TVM_RUNTIME_CRT_INTERNAL_GRAPH_RUNTIME_LOAD_JSON_H_ +#define TVM_RUNTIME_CRT_INCLUDE_TVM_RUNTIME_CRT_INTERNAL_GRAPH_RUNTIME_LOAD_JSON_H_ #include +#include #include enum { @@ -89,4 +90,4 @@ JSONReader JSONReader_Create(const char* is); void JSONReader_Release(JSONReader* reader); -#endif // TVM_RUNTIME_CRT_LOAD_JSON_H_ +#endif // TVM_RUNTIME_CRT_INCLUDE_TVM_RUNTIME_CRT_INTERNAL_GRAPH_RUNTIME_LOAD_JSON_H_ diff --git a/src/runtime/library_module.cc b/src/runtime/library_module.cc index 7c3323c56229..b12a9d195e2e 100644 --- a/src/runtime/library_module.cc +++ b/src/runtime/library_module.cc @@ -74,7 +74,7 @@ PackedFunc WrapPackedFunc(TVMBackendPackedCFunc faddr, const ObjectPtr& TVMValue ret_value; int ret_type_code = kTVMNullptr; int ret = (*faddr)(const_cast(args.values), const_cast(args.type_codes), - args.num_args, &ret_value, &ret_type_code); + args.num_args, &ret_value, &ret_type_code, NULL); CHECK_EQ(ret, 0) << TVMGetLastError(); if (ret_type_code != kTVMNullptr) { *rv = TVMRetValue::MoveFromCHost(ret_value, ret_type_code); diff --git a/src/tir/transforms/make_packed_api.cc b/src/tir/transforms/make_packed_api.cc index 191bb0a60554..9519fa6cebe1 100644 --- a/src/tir/transforms/make_packed_api.cc +++ b/src/tir/transforms/make_packed_api.cc @@ -68,6 +68,7 @@ PrimFunc MakePackedAPI(PrimFunc&& func, int num_unpacked_args) { Var v_num_packed_args("num_args", DataType::Int(32)); Var v_out_ret_value("out_ret_value", DataType::Handle()); Var v_out_ret_tcode("out_ret_tcode", DataType::Handle()); + Var v_resource_handle("resource_handle", DataType::Handle()); // The arguments of the function. Array args; // The device context @@ -156,9 +157,10 @@ PrimFunc MakePackedAPI(PrimFunc&& func, int num_unpacked_args) { if (num_packed_args != 0) { args.push_back(v_out_ret_value); args.push_back(v_out_ret_tcode); + args.push_back(v_resource_handle); } - size_t expected_nargs = num_unpacked_args + (num_packed_args != 0 ? 5 : 0); + size_t expected_nargs = num_unpacked_args + (num_packed_args != 0 ? 6 : 0); CHECK_EQ(args.size(), expected_nargs); // Arg definitions are defined before buffer binding to avoid the use before diff --git a/tests/crt/func_registry_test.cc b/tests/crt/func_registry_test.cc new file mode 100644 index 000000000000..2eca2a3dcd6b --- /dev/null +++ b/tests/crt/func_registry_test.cc @@ -0,0 +1,238 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include +#include +#include +#include + +typedef struct { + const char* a; + const char* b; + int ret_val; +} strcmp_test_t; + +strcmp_test_t strcmp_tests[] = { + {"Foo", "Foo", 0}, {"Foo", "Bar", 'F' - 'B'}, {"Foo", "", 'F'}, + {"Fabulous", "Fab", 'u'}, {"Fab", "Fabulous", 0 - 'u'}, +}; + +std::ostream& operator<<(std::ostream& os, const strcmp_test_t& test) { + os << "strcmp_cursor(\"" << test.a << "\", \"" << test.b << "\") -> " << test.ret_val; + return os; +} + +class StrCmpTestFixture : public ::testing::TestWithParam {}; + +TEST_P(StrCmpTestFixture, Match) { + strcmp_test_t param = GetParam(); + const char* cursor = param.a; + EXPECT_EQ(param.ret_val, strcmp_cursor(&cursor, param.b)); + + EXPECT_EQ('\0', *cursor); + + size_t a_length = strlen(param.a); + EXPECT_EQ(param.a + a_length, cursor); +} + +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wdeprecated-declarations" +INSTANTIATE_TEST_CASE_P(StrCmpTests, StrCmpTestFixture, ::testing::ValuesIn(strcmp_tests)); +#pragma GCC diagnostic pop + +TEST(StrCmpScan, Test) { + const char* a = "Foo\0Bar\0Whoops\0"; + const char* cursor = a; + + EXPECT_EQ('o', strcmp_cursor(&cursor, "Fo")); + EXPECT_EQ(0, *cursor); + EXPECT_EQ(cursor, a + 3); + cursor++; + + EXPECT_EQ(0 - 'r', strcmp_cursor(&cursor, "Barr")); + EXPECT_EQ(0, *cursor); + EXPECT_EQ(cursor, a + 7); + cursor++; + + EXPECT_EQ('h' - 'B', strcmp_cursor(&cursor, "WB")); + EXPECT_EQ(0, *cursor); + EXPECT_EQ(cursor, a + 14); + cursor++; + + EXPECT_EQ(0, *cursor); + const char* before_cursor = cursor; + EXPECT_EQ(0, strcmp_cursor(&cursor, "")); + EXPECT_EQ(before_cursor, cursor); +} + +TEST(FuncRegistry, Empty) { + TVMFuncRegistry registry{"\000", NULL}; + + EXPECT_EQ(kTvmErrorFunctionNameNotFound, TVMFuncRegistry_Lookup(®istry, "foo", NULL)); + EXPECT_EQ(kTvmErrorFunctionIndexInvalid, + TVMFuncRegistry_GetByIndex(®istry, (tvm_function_index_t)0, NULL)); +} + +extern "C" { +static int Foo(TVMValue* args, int* type_codes, int num_args, TVMValue* out_ret_value, + int* out_ret_tcode, void* resource_handle) { + return 0; +} +static int Bar(TVMValue* args, int* type_codes, int num_args, TVMValue* out_ret_value, + int* out_ret_tcode, void* resource_handle) { + return 0; +} +} + +// Matches the style of registry defined in generated C modules. +const char* kBasicFuncNames = "\002Foo\0Bar\0"; // NOTE: final \0 +const TVMBackendPackedCFunc funcs[2] = {&Foo, &Bar}; +const TVMFuncRegistry kConstRegistry = {kBasicFuncNames, (const TVMBackendPackedCFunc*)funcs}; + +TEST(FuncRegistry, ConstGlobalRegistry) { + tvm_function_index_t func_index = -1; + TVMBackendPackedCFunc func = nullptr; + + // Foo + EXPECT_EQ(kBasicFuncNames[0], 2); + EXPECT_EQ(kBasicFuncNames[1], 'F'); + EXPECT_EQ(kTvmErrorNoError, TVMFuncRegistry_Lookup(&kConstRegistry, "Foo", &func_index)); + EXPECT_EQ(0, func_index); + + EXPECT_EQ(kTvmErrorNoError, TVMFuncRegistry_GetByIndex(&kConstRegistry, func_index, &func)); + EXPECT_EQ(func, &Foo); + + // Bar + EXPECT_EQ(kTvmErrorNoError, TVMFuncRegistry_Lookup(&kConstRegistry, "Bar", &func_index)); + EXPECT_EQ(1, func_index); + + EXPECT_EQ(kTvmErrorNoError, TVMFuncRegistry_GetByIndex(&kConstRegistry, func_index, &func)); + EXPECT_EQ(func, &Bar); + + // Expected not found. + tvm_function_index_t prev_func_index = func_index; + EXPECT_EQ(kTvmErrorFunctionNameNotFound, + TVMFuncRegistry_Lookup(&kConstRegistry, "Baz", &func_index)); + EXPECT_EQ(prev_func_index, func_index); + + // Expected index out of range. + func = nullptr; + EXPECT_EQ(kTvmErrorFunctionIndexInvalid, TVMFuncRegistry_GetByIndex(&kConstRegistry, 2, &func)); + EXPECT_EQ(func, nullptr); +} + +/*! \brief Return a test function handle, with number repeating for all bytes in a void*. */ +static TVMBackendPackedCFunc TestFunctionHandle(uint8_t number) { + uintptr_t handle = 0; + for (size_t i = 0; i < sizeof(TVMBackendPackedCFunc); i++) { + handle |= ((uintptr_t)handle) << (8 * i); + } + + return (TVMBackendPackedCFunc)handle; +} + +static void snprintf_truncate(char* target, size_t bytes, const char* str) { +#ifdef __GNUC__ +#if __GNUC__ > 7 || (__GNUC__ == 7 && __GNUC_MINOR__ >= 1) +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wformat-truncation" +#endif +#endif + EXPECT_GT(snprintf(target, bytes, "%s", str), 0); +#ifdef __GNUC__ +#if __GNUC__ > 7 || (__GNUC__ == 7 && __GNUC_MINOR__ >= 1) +#pragma GCC diagnostic pop +#endif +#endif +} + +TEST(MutableFuncRegistry, Create) { + uint8_t mem_buffer[kTvmAverageFuncEntrySizeBytes * 3]; + // A substring used to create function names for testing. + const char* function_name_chars = "abcdefghijklmnopqrstuvwxyzyxw"; + + // function_name_chars is used to produce 2 function names. The second one is expected to + // overfill `names`; assert there are at least enough data in function_name_chars to do this. + EXPECT_LE(kTvmAverageFuncEntrySizeBytes + kTvmAverageFunctionNameStrlenBytes, + strlen(function_name_chars)); + + for (unsigned int buf_size = 0; buf_size < kTvmAverageFuncEntrySizeBytes; buf_size++) { + EXPECT_EQ(kTvmErrorBufferTooSmall, TVMMutableFuncRegistry_Create(NULL, mem_buffer, buf_size)); + } + + for (unsigned int rem = 0; rem < kTvmAverageFuncEntrySizeBytes; rem++) { + // test_function name will be used to test overfilling. + char test_function_name[kTvmAverageFunctionNameStrlenBytes + 2 + rem]; + TVMMutableFuncRegistry reg; + memset(mem_buffer, 0, sizeof(mem_buffer)); + EXPECT_EQ(kTvmErrorNoError, TVMMutableFuncRegistry_Create( + ®, mem_buffer, kTvmAverageFuncEntrySizeBytes * 2 + rem)); + + snprintf_truncate(test_function_name, kTvmAverageFunctionNameStrlenBytes + 1, + function_name_chars); + + // Add function #1, and verify it can be retrieved. + EXPECT_EQ(kTvmErrorNoError, + TVMMutableFuncRegistry_Set(®, test_function_name, TestFunctionHandle(0x01), 0)); + + tvm_function_index_t func_index = 100; + EXPECT_EQ(kTvmErrorNoError, + TVMFuncRegistry_Lookup(®.registry, test_function_name, &func_index)); + EXPECT_EQ(func_index, 0); + + TVMBackendPackedCFunc func = NULL; + EXPECT_EQ(kTvmErrorNoError, TVMFuncRegistry_GetByIndex(®.registry, func_index, &func)); + EXPECT_EQ(func, TestFunctionHandle(0x01)); + + // Ensure that overfilling `names` by 1 char is not allowed. + snprintf_truncate(test_function_name, kTvmAverageFunctionNameStrlenBytes + rem + 2, + function_name_chars + 1); + + EXPECT_EQ(kTvmErrorFunctionRegistryFull, + TVMMutableFuncRegistry_Set(®, test_function_name, TestFunctionHandle(0x02), 0)); + EXPECT_EQ(kTvmErrorFunctionNameNotFound, + TVMFuncRegistry_Lookup(®.registry, test_function_name, &func_index)); + + // Add function #2, with intentionally short (by 2 char) name. Verify it can be retrieved. + snprintf_truncate(test_function_name, kTvmAverageFunctionNameStrlenBytes - 2 + 1, + function_name_chars + 1); + EXPECT_EQ(kTvmErrorNoError, + TVMMutableFuncRegistry_Set(®, test_function_name, TestFunctionHandle(0x02), 0)); + + EXPECT_EQ(kTvmErrorNoError, + TVMFuncRegistry_Lookup(®.registry, test_function_name, &func_index)); + EXPECT_EQ(func_index, 1); + + func = NULL; + EXPECT_EQ(kTvmErrorNoError, TVMFuncRegistry_GetByIndex(®.registry, func_index, &func)); + EXPECT_EQ(func, TestFunctionHandle(0x01)); + + // Try adding another function, which should fail due to lack of function pointers. + test_function_name[0] = 'a'; + test_function_name[1] = 0; + EXPECT_EQ(kTvmErrorFunctionRegistryFull, + TVMMutableFuncRegistry_Set(®, test_function_name, TestFunctionHandle(0x03), 0)); + } +} + +int main(int argc, char** argv) { + testing::InitGoogleTest(&argc, argv); + testing::FLAGS_gtest_death_test_style = "threadsafe"; + return RUN_ALL_TESTS(); +} diff --git a/tests/crt/memory_test.cc b/tests/crt/memory_test.cc new file mode 100644 index 000000000000..3b1f7fa560fe --- /dev/null +++ b/tests/crt/memory_test.cc @@ -0,0 +1,130 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include +#include +#include + +#include "crt_config.h" + +#define ROUND_UP(qty, modulo) (((qty) + ((modulo)-1)) / (modulo) * (modulo)) + +static constexpr const unsigned int kTotalPages = 128; +static constexpr const unsigned int kNumUsablePages = + (sizeof(void*) == 8 ? 95 : (sizeof(void*) == 4 ? 99 : 0)); +static constexpr const unsigned int kPageSizeBytesLog = 8; // 256 byte pages. +static constexpr const unsigned int kMemoryPoolSizeBytes = kTotalPages * (1 << kPageSizeBytesLog); + +class MemoryManagerTest : public ::testing::Test { + protected: + void SetUp() override { + memset(raw_memory_pool, 0, sizeof(raw_memory_pool)); + memory_pool = (uint8_t*)(ROUND_UP(((uintptr_t)raw_memory_pool), (1 << kPageSizeBytesLog))); + MemoryManagerCreate(&mgr, memory_pool, kMemoryPoolSizeBytes, kPageSizeBytesLog); + ASSERT_EQ(kNumUsablePages, mgr.ptable.max_pages); + } + + unsigned int AddressToPageNumber(void* a) { + return (reinterpret_cast(a) - reinterpret_cast(memory_pool)) >> + kPageSizeBytesLog; + } + + uint8_t raw_memory_pool[kMemoryPoolSizeBytes + (1 << kPageSizeBytesLog)]; + uint8_t* memory_pool; + MemoryManager mgr; +}; + +#define EXPECT_PAGE(expected, actual) EXPECT_EQ(expected, AddressToPageNumber(actual)) + +TEST_F(MemoryManagerTest, AllocFreeFifo) { + EXPECT_EQ(vleak_size, 0); + + for (int i = 0; i < 2; i++) { + void* ptrs[kNumUsablePages]; + for (size_t idx = 0; idx < kNumUsablePages; idx++) { + void* a = mgr.Alloc(&mgr, 1); + if (i == 0) { + EXPECT_PAGE(idx, a); + } else { + EXPECT_PAGE(kNumUsablePages - 1 - idx, a); + } + EXPECT_EQ(vleak_size, idx + 1); + ptrs[idx] = a; + } + + for (int idx = kNumUsablePages - 1; idx >= 0; idx--) { + mgr.Free(&mgr, ptrs[idx]); + EXPECT_EQ(vleak_size, idx); + } + } +} + +TEST_F(MemoryManagerTest, Realloc) { + EXPECT_EQ(vleak_size, 0); + + void* a = mgr.Realloc(&mgr, 0, 1); + EXPECT_PAGE(0, a); + EXPECT_EQ(vleak_size, 1); + + void* b = mgr.Realloc(&mgr, a, 50); + EXPECT_PAGE(0, b); + EXPECT_EQ(vleak_size, 1); + + void* c = mgr.Realloc(&mgr, b, 50 + (1 << kPageSizeBytesLog)); + EXPECT_PAGE(1, c); + EXPECT_EQ(vleak_size, 1); + + void* d = mgr.Alloc(&mgr, 30); + EXPECT_PAGE(0, d); + EXPECT_EQ(vleak_size, 2); + + void* e = mgr.Realloc(&mgr, c, (50 + (2 << kPageSizeBytesLog))); + EXPECT_PAGE(3, e); + EXPECT_EQ(vleak_size, 2); + + void* f = mgr.Alloc(&mgr, 30); + EXPECT_PAGE(1, f); + EXPECT_EQ(vleak_size, 3); + + mgr.Free(&mgr, f); + EXPECT_EQ(vleak_size, 2); + + mgr.Free(&mgr, e); + EXPECT_EQ(vleak_size, 1); + + mgr.Free(&mgr, e); + EXPECT_EQ(vleak_size, 0); + + void* g = mgr.Alloc(&mgr, 1); + EXPECT_PAGE(1, g); + EXPECT_EQ(vleak_size, 1); + + mgr.Free(&mgr, g); + EXPECT_EQ(vleak_size, 0); +} + +extern "C" { +void TVMPlatformAbort(int error_code) { FAIL() << "TVMPlatformAbort(" << error_code << ")"; } +} + +int main(int argc, char** argv) { + testing::InitGoogleTest(&argc, argv); + testing::FLAGS_gtest_death_test_style = "threadsafe"; + return RUN_ALL_TESTS(); +} diff --git a/tests/lint/git-clang-format.sh b/tests/lint/git-clang-format.sh index b1ae1bced439..90f18357fabe 100755 --- a/tests/lint/git-clang-format.sh +++ b/tests/lint/git-clang-format.sh @@ -19,6 +19,13 @@ set -e set -u set -o pipefail +if [[ "$1" == "-i" ]]; then + INPLACE_FORMAT=1 + shift 1 +else + INPLACE_FORMAT=0 +fi + if [[ "$#" -lt 1 ]]; then echo "Usage: tests/lint/git-clang-format.sh [-i] " echo "" @@ -30,13 +37,6 @@ if [[ "$#" -lt 1 ]]; then exit 1 fi -if [[ "$1" == "-i" ]]; then - INPLACE_FORMAT=1 - shift 1 -else - INPLACE_FORMAT=0 -fi - cleanup() { rm -rf /tmp/$$.clang-format.txt diff --git a/tests/python/unittest/test_tir_transform_make_packed_api.py b/tests/python/unittest/test_tir_transform_make_packed_api.py index 760cf477f959..161745c68e7d 100644 --- a/tests/python/unittest/test_tir_transform_make_packed_api.py +++ b/tests/python/unittest/test_tir_transform_make_packed_api.py @@ -39,7 +39,7 @@ def test_makeapi(): num_unpacked_args = 2 f = tvm.tir.transform.MakePackedAPI(num_unpacked_args)(mod)["main"] - assert(len(f.params) == 7) + assert(len(f.params) == 8) if __name__ == "__main__": diff --git a/tests/scripts/task_config_build_cpu.sh b/tests/scripts/task_config_build_cpu.sh index d64bcab7bc8f..529b9965eb7d 100755 --- a/tests/scripts/task_config_build_cpu.sh +++ b/tests/scripts/task_config_build_cpu.sh @@ -26,6 +26,7 @@ cp ../cmake/config.cmake . echo set\(USE_SORT ON\) >> config.cmake echo set\(USE_MICRO ON\) >> config.cmake echo set\(USE_MICRO_STANDALONE_RUNTIME ON\) >> config.cmake +echo set\(USE_STANDALONE_CRT ON\) >> config.cmake echo set\(USE_GRAPH_RUNTIME_DEBUG ON\) >> config.cmake echo set\(USE_VM_PROFILER ON\) >> config.cmake echo set\(USE_EXAMPLE_EXT_RUNTIME ON\) >> config.cmake diff --git a/tests/scripts/task_config_build_gpu.sh b/tests/scripts/task_config_build_gpu.sh index c7b073728b70..2852fdef47dc 100755 --- a/tests/scripts/task_config_build_gpu.sh +++ b/tests/scripts/task_config_build_gpu.sh @@ -29,6 +29,7 @@ echo set\(USE_CUDA ON\) >> config.cmake echo set\(USE_OPENGL ON\) >> config.cmake echo set\(USE_MICRO ON\) >> config.cmake echo set\(USE_MICRO_STANDALONE_RUNTIME ON\) >> config.cmake +echo set\(USE_STANDALONE_CRT ON\) >> config.cmake echo set\(USE_LLVM llvm-config-9\) >> config.cmake echo set\(USE_NNPACK ON\) >> config.cmake echo set\(NNPACK_PATH /NNPACK/build/\) >> config.cmake diff --git a/tests/scripts/task_config_build_gpu_vulkan.sh b/tests/scripts/task_config_build_gpu_vulkan.sh index 2627c1f48e18..e07f97d0efa0 100755 --- a/tests/scripts/task_config_build_gpu_vulkan.sh +++ b/tests/scripts/task_config_build_gpu_vulkan.sh @@ -27,6 +27,7 @@ echo set\(USE_OPENCL ON\) >> config.cmake echo set\(USE_ROCM ON\) >> config.cmake echo set\(USE_VULKAN ON\) >> config.cmake echo set\(USE_MICRO ON\) >> config.cmake +echo set\(USE_STANDALONE_CRT ON\) >> config.cmake echo set\(USE_GRAPH_RUNTIME_DEBUG ON\) >> config.cmake echo set\(USE_VM_PROFILER ON\) >> config.cmake echo set\(USE_EXAMPLE_EXT_RUNTIME ON\) >> config.cmake diff --git a/tests/scripts/task_config_build_i386.sh b/tests/scripts/task_config_build_i386.sh index e5ad56c7afdf..6837c2830091 100755 --- a/tests/scripts/task_config_build_i386.sh +++ b/tests/scripts/task_config_build_i386.sh @@ -27,6 +27,7 @@ echo set\(USE_SORT ON\) >> config.cmake echo set\(USE_RPC ON\) >> config.cmake echo set\(USE_GRAPH_RUNTIME_DEBUG ON\) >> config.cmake echo set\(USE_MICRO_STANDALONE_RUNTIME ON\) >> config.cmake +echo set\(USE_STANDALONE_CRT ON\) >> config.cmake echo set\(USE_VM_PROFILER ON\) >> config.cmake echo set\(USE_EXAMPLE_EXT_RUNTIME ON\) >> config.cmake echo set\(USE_LLVM llvm-config-4.0\) >> config.cmake diff --git a/tests/scripts/task_config_build_wasm.sh b/tests/scripts/task_config_build_wasm.sh index cf388eb2fbdc..cbdfa75e456e 100755 --- a/tests/scripts/task_config_build_wasm.sh +++ b/tests/scripts/task_config_build_wasm.sh @@ -26,6 +26,7 @@ cp ../cmake/config.cmake . echo set\(USE_SORT ON\) >> config.cmake echo set\(USE_MICRO ON\) >> config.cmake echo set\(USE_MICRO_STANDALONE_RUNTIME ON\) >> config.cmake +echo set\(USE_STANDALONE_CRT ON\) >> config.cmake echo set\(USE_GRAPH_RUNTIME_DEBUG ON\) >> config.cmake echo set\(USE_VM_PROFILER ON\) >> config.cmake echo set\(USE_EXAMPLE_EXT_RUNTIME ON\) >> config.cmake diff --git a/tests/scripts/task_cpp_unittest.sh b/tests/scripts/task_cpp_unittest.sh index 5ac1843253d4..712e54a1a240 100755 --- a/tests/scripts/task_cpp_unittest.sh +++ b/tests/scripts/task_cpp_unittest.sh @@ -30,7 +30,7 @@ export OMP_NUM_THREADS=1 # Remove existing testcases rm -f build/*_test -make cpptest -j8 +make crttest cpptest -j3 for test in build/*_test; do ./$test done