[uTVM][Runtime] Introduce Virtual Memory Allocator to CRT (#5124)

* initial crt_memory and memory leak fix in graph_runtime

Change-Id: I0f79f909a04d1c677aabb80f202f0612c5ce7f2a

* fix memory leak

Change-Id: I37104c09e28112b1974fa2b064c809d0a8d686c3

* clean up

Change-Id: I039b12015a1d56c8f4120867cd5a5292da34f3e3

* implement vrealloc

Change-Id: I35800470bcbfcf96652494f359711cb4c2d34398

* allocate from stack memory for most of the variables

Change-Id: I72071289843fff4031c0df8796868a0b9fbc57ee

* allocate from stack memory for all of the variables

Change-Id: I32dba85ac1660c77f51c2d0d8ab6436ed0c01c74

* lint

Change-Id: If12cd240685d7791fc60bc0cfb66389cdc186b73

* lint

Change-Id: I7c9d90c11b60b8edda2427ebd189ebe535af2100

* facilitate the growth of TVM_CRT_MAX_NDIM

Change-Id: I939fa43027a5c7529c5c7c6bd8d6e6beb91b7581

* extend test coverage of vmalloc

Change-Id: Ie4ff6b64fdfe6810836cf8fd44dace82a20c4581

* lint

Change-Id: Ibf3c06619ef296df5c49f3945cb6428777781d69

* move logging.h to src

* fix an error in macOS

* remove logging.h

* use cflags for gcc

* fix compilation error

apps/bundle_deploy/Makefile

@@ -20,11 +20,11 @@
 # Setup build environment
 TVM_ROOT=$(shell cd ../..; pwd)
 DMLC_CORE=${TVM_ROOT}/3rdparty/dmlc-core
-PKG_CXXFLAGS = -std=c++14 -O2 -fPIC \
+PKG_CXXFLAGS = -Wall -std=c++14 -O2 -fPIC \
 	-I${TVM_ROOT}/include \
 	-I${DMLC_CORE}/include \
 	-I${TVM_ROOT}/3rdparty/dlpack/include
-PKG_CFLAGS = -std=c99 -O2 -fPIC \
+PKG_CFLAGS = -Wall -std=c99 -O2 -fPIC \
 	-I${TVM_ROOT}/include \
 	-I${DMLC_CORE}/include \
 	-I${TVM_ROOT}/3rdparty/dlpack/include
@@ -57,11 +57,11 @@ $(build_dir)/test_dynamic: test.cc ${build_dir}/test_graph.json ${build_dir}/tes
 
 $(build_dir)/demo_static: demo_static.c ${build_dir}/bundle_static.o ${build_dir}/model.o ${build_dir}/graph.json.c ${build_dir}/params.bin.c
 	@mkdir -p $(@D)
-	gcc $(PKG_CXXFLAGS) -o $@ demo_static.c ${build_dir}/bundle_static.o ${build_dir}/model.o -lm
+	gcc $(PKG_CFLAGS) -o $@ demo_static.c ${build_dir}/bundle_static.o ${build_dir}/model.o -lm
 
 $(build_dir)/test_static: test_static.c ${build_dir}/bundle_static.o ${build_dir}/test_model.o
 	@mkdir -p $(@D)
-	gcc $(PKG_CXXFLAGS) -o $@ $^
+	gcc $(PKG_CFLAGS) -o $@ $^
 
 # Serialize our graph.json file.
 $(build_dir)/graph.json.c: $(build_dir)/graph.json
@@ -71,14 +71,6 @@ $(build_dir)/graph.json.c: $(build_dir)/graph.json
 $(build_dir)/params.bin.c: $(build_dir)/params.bin
 	xxd -i $^  > $@
 
-# # Serialize our test_graph.json file.
-# $(build_dir)/test_graph.json.c: $(build_dir)/test_graph.json
-# 	xxd -i $^  > $@
-# 
-# # Serialize our test_params.bin file.
-# $(build_dir)/test_params.bin.c: $(build_dir)/test_params.bin
-# 	xxd -i $^  > $@
-
 $(build_dir)/model.o $(build_dir)/graph.json $(build_dir)/params.bin $(build_dir)/cat.bin: build_model.py
 	python3 $< -o $(build_dir)
 

apps/bundle_deploy/demo.cc

@@ -109,11 +109,11 @@ int main(int argc, char **argv) {
          max_index, max_iter);
   printf("timing: %.2f ms (create), %.2f ms (set_input), %.2f ms (run), "
          "%.2f ms (get_output), %.2f ms (destroy)\n",
-         (t1.tv_sec-t0.tv_sec)*1000000 + (t1.tv_usec-t0.tv_usec)/1000.f,
-         (t2.tv_sec-t1.tv_sec)*1000000 + (t2.tv_usec-t1.tv_usec)/1000.f,
-         (t3.tv_sec-t2.tv_sec)*1000000 + (t3.tv_usec-t2.tv_usec)/1000.f,
-         (t4.tv_sec-t3.tv_sec)*1000000 + (t4.tv_usec-t3.tv_usec)/1000.f,
-         (t5.tv_sec-t4.tv_sec)*1000000 + (t5.tv_usec-t4.tv_usec)/1000.f);
+         (t1.tv_sec-t0.tv_sec)*1000.0f + (t1.tv_usec-t0.tv_usec)/1000.f,
+         (t2.tv_sec-t1.tv_sec)*1000.0f + (t2.tv_usec-t1.tv_usec)/1000.f,
+         (t3.tv_sec-t2.tv_sec)*1000.0f + (t3.tv_usec-t2.tv_usec)/1000.f,
+         (t4.tv_sec-t3.tv_sec)*1000.0f + (t4.tv_usec-t3.tv_usec)/1000.f,
+         (t5.tv_sec-t4.tv_sec)*1000.0f + (t5.tv_usec-t4.tv_usec)/1000.f);
   dlclose(bundle);
 
   return 0;

apps/bundle_deploy/runtime.c

@@ -30,23 +30,37 @@
 #define TVM_CRT_MAX_NDIM 6
 /*! Maximum supported arguments in generated functions */
 #define TVM_CRT_MAX_ARGS 10
+/*! Maximum supported string length in dltype, e.g. "int8", "int16", "float32" */
+#define TVM_CRT_STRLEN_DLTYPE 10
+/*! Maximum supported string length in function names */
+#define TVM_CRT_STRLEN_NAME 80
 
-/*! Maximum inputs in a GraphRuntimeNode */
-#define GRAPH_RUNTIME_NODE_MAX_INPUTS 300
-/*! Maximum supported contexts in a GraphRuntime */
-#define GRAPH_RUNTIME_MAX_CONTEXTS 1
-/*! Maximum supported nodes in a GraphRuntime */
-#define GRAPH_RUNTIME_MAX_NODES 400
-/*! Maximum input nodes in a GraphRuntime */
-#define GRAPH_RUNTIME_MAX_INPUT_NODES 300
-/*! Maximum nodes in a GraphRuntime for quick entry indexing */
-#define GRAPH_RUNTIME_MAX_NODE_ROW_PTR 300
-/*! Maximum output entries in a GraphRuntime */
-#define GRAPH_RUNTIME_MAX_OUTPUTS 300
+/*!
+ * \brief Log memory pool size for virtual memory allocation
+ *
+ * Here is a list of possible choices:
+ * * use 16 for 64 KiB memory space
+ * * use 17 for 128 KiB memory space
+ * * use 18 for 256 KiB memory space
+ * * use 19 for 512 KiB memory space
+ * * use 20 for 1 MiB memory space
+ * * use 21 for 2 MiB memory space
+ * * use 22 for 4 MiB memory space
+ * * use 23 for 8 MiB memory space
+ * * use 24 for 16 MiB memory space
+ * * use 25 for 32 MiB memory space
+ * * use 26 for 64 MiB memory space
+ * * use 27 for 128 MiB memory space
+ * * use 28 for 256 MiB memory space
+ */
+#define TVM_CRT_LOG_VIRT_MEM_SIZE 24
+
+/*! \brief Page size for virtual memory allocation */
+#define TVM_CRT_PAGE_BYTES 4096
 
 #include "../../src/runtime/crt/crt_runtime_api.c"
 #include "../../src/runtime/crt/crt_backend_api.c"
 #include "../../src/runtime/crt/graph_runtime.c"
 #include "../../src/runtime/crt/load_json.c"
 #include "../../src/runtime/crt/ndarray.c"
-
+#include "../../src/runtime/crt/memory.c"

apps/bundle_deploy/test.cc

@@ -122,11 +122,11 @@ int main(int argc, char **argv) {
 
   printf("timing: %.2f ms (create), %.2f ms (set_input), %.2f ms (run), "
          "%.2f ms (get_output), %.2f ms (destroy)\n",
-         (t1.tv_sec-t0.tv_sec)*1000000 + (t1.tv_usec-t0.tv_usec)/1000.f,
-         (t2.tv_sec-t1.tv_sec)*1000000 + (t2.tv_usec-t1.tv_usec)/1000.f,
-         (t3.tv_sec-t2.tv_sec)*1000000 + (t3.tv_usec-t2.tv_usec)/1000.f,
-         (t4.tv_sec-t3.tv_sec)*1000000 + (t4.tv_usec-t3.tv_usec)/1000.f,
-         (t5.tv_sec-t4.tv_sec)*1000000 + (t5.tv_usec-t4.tv_usec)/1000.f);
+         (t1.tv_sec-t0.tv_sec)*1000.0f + (t1.tv_usec-t0.tv_usec)/1000.f,
+         (t2.tv_sec-t1.tv_sec)*1000.0f + (t2.tv_usec-t1.tv_usec)/1000.f,
+         (t3.tv_sec-t2.tv_sec)*1000.0f + (t3.tv_usec-t2.tv_usec)/1000.f,
+         (t4.tv_sec-t3.tv_sec)*1000.0f + (t4.tv_usec-t3.tv_usec)/1000.f,
+         (t5.tv_sec-t4.tv_sec)*1000.0f + (t5.tv_usec-t4.tv_usec)/1000.f);
 
   free(json_data);
   free(params_data);

include/tvm/runtime/crt/memory.h

@@ -0,0 +1,52 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/*!
+ * \file tvm/runtime/crt/memory.h
+ * \brief The virtual memory manager for micro-controllers
+ */
+
+#ifndef TVM_RUNTIME_CRT_MEMORY_H_
+#define TVM_RUNTIME_CRT_MEMORY_H_
+
+static int vleak_size = 0;
+
+/*!
+ * \brief Allocate memory from manager
+ * \param size The size of memory
+ * \return The virtual address
+ */
+void * vmalloc(size_t size);
+
+/*!
+ * \brief Reallocate memory from manager
+ * \param ptr The pointer to the memory area to be reallocated
+ * \param size The size of memory
+ * \return The virtual address
+ */
+void * vrealloc(void * ptr, size_t size);
+
+/*!
+ * \brief Free the memory.
+ * \param ptr The pointer to the memory to deallocate
+ * \return The virtual address
+ */
+void vfree(void * ptr);
+
+#endif  // TVM_RUNTIME_CRT_MEMORY_H_

src/runtime/crt/crt_backend_api.c

@@ -18,6 +18,7 @@
  */
 
 #include <tvm/runtime/c_backend_api.h>
+#include <tvm/runtime/crt/memory.h>
 
 #include <stdio.h>
 #include <stdlib.h>
@@ -29,18 +30,12 @@ void* TVMBackendAllocWorkspace(int device_type, int device_id, uint64_t nbytes,
   void* ptr = 0;
   assert(nbytes > 0);
   unsigned int dtype_bytes = dtype_bits_hint / 8;
-#ifdef __ANDROID__
-  ptr = memalign(64, nbytes * dtype_bytes);
-#else
-  const int ret = posix_memalign(&ptr, 64, nbytes * dtype_bytes);
-  (void)ret;
-  assert(ret == 0);
-#endif
+  ptr = vmalloc(nbytes * dtype_bytes);
   return ptr;
 }
 
 int TVMBackendFreeWorkspace(int device_type, int device_id, void* ptr) {
-  free(ptr);
+  vfree(ptr);
   return 0;
 }
 
@@ -52,6 +47,7 @@ int TVMBackendParallelLaunch(FTVMParallelLambda flambda, void* cdata, int num_ta
 }
 
 int TVMBackendRegisterSystemLibSymbol(const char* name, void* ptr) {
+  g_fexecs = vrealloc(g_fexecs, sizeof(TVMPackedFunc) * (g_fexecs_count + 1));
   snprintf(g_fexecs[g_fexecs_count].name, sizeof(g_fexecs[g_fexecs_count].name), name);
   g_fexecs[g_fexecs_count].fexec = ptr;
   g_fexecs_count++;