From e88fe7726f58c415ac825b3a39ec0e034d3fa009 Mon Sep 17 00:00:00 2001 From: Mehrdad Hessar Date: Wed, 11 Aug 2021 16:24:54 -0700 Subject: [PATCH] [microTVM] Zephyr Test Refactor (#8713) * refactor host to qemu * remove unused variables * remove skip-build arg * fix microtvm test script --- tests/micro/zephyr/conftest.py | 18 +++------- tests/micro/zephyr/test_zephyr.py | 47 ++++++++++++--------------- tests/micro/zephyr/test_zephyr_aot.py | 21 ++++-------- tests/scripts/task_python_microtvm.sh | 2 +- 4 files changed, 31 insertions(+), 57 deletions(-) diff --git a/tests/micro/zephyr/conftest.py b/tests/micro/zephyr/conftest.py index 2b30401a90e9..cfdb208c92b8 100644 --- a/tests/micro/zephyr/conftest.py +++ b/tests/micro/zephyr/conftest.py @@ -26,9 +26,9 @@ # The models that should pass this configuration. Maps a short, identifying platform string to # (model, zephyr_board). PLATFORMS = { - "host": ("host", "qemu_x86"), - "host_riscv32": ("host", "qemu_riscv32"), - "host_riscv64": ("host", "qemu_riscv64"), + "qemu_x86": ("host", "qemu_x86"), + "qemu_riscv32": ("host", "qemu_riscv32"), + "qemu_riscv64": ("host", "qemu_riscv64"), "mps2_an521": ("mps2_an521", "mps2_an521"), "nrf5340dk": ("nrf5340dk", "nrf5340dk_nrf5340_cpuapp"), "stm32f746xx_disco": ("stm32f746xx", "stm32f746g_disco"), @@ -41,7 +41,7 @@ def pytest_addoption(parser): parser.addoption( "--microtvm-platforms", - default="host", + default="qemu_x86", choices=PLATFORMS.keys(), help=( "Specify a comma-separated list of test models (i.e. as passed to tvm.target.micro()) " @@ -51,11 +51,6 @@ def pytest_addoption(parser): parser.addoption( "--west-cmd", default="west", help="Path to `west` command for flashing device." ) - parser.addoption( - "--skip-build", - action="store_true", - help="If set true, reuses build from the previous test run. Otherwise, build from the scratch.", - ) parser.addoption( "--tvm-debug", action="store_true", @@ -74,11 +69,6 @@ def west_cmd(request): return request.config.getoption("--west-cmd") -@pytest.fixture -def skip_build(request): - return request.config.getoption("--skip-build") - - @pytest.fixture def tvm_debug(request): return request.config.getoption("--tvm-debug") diff --git a/tests/micro/zephyr/test_zephyr.py b/tests/micro/zephyr/test_zephyr.py index b84a15225f7a..14e250c3a44e 100644 --- a/tests/micro/zephyr/test_zephyr.py +++ b/tests/micro/zephyr/test_zephyr.py @@ -57,7 +57,7 @@ def _make_sess_from_op( with tvm.transform.PassContext(opt_level=3, config={"tir.disable_vectorize": True}): mod = tvm.build(sched, arg_bufs, target=target, name=op_name) - return _make_session(temp_dir, model, target, zephyr_board, west_cmd, mod, build_config) + return _make_session(temp_dir, zephyr_board, west_cmd, mod, build_config) TEMPLATE_PROJECT_DIR = ( @@ -72,7 +72,7 @@ def _make_sess_from_op( ).resolve() -def _make_session(temp_dir, model, target, zephyr_board, west_cmd, mod, build_config): +def _make_session(temp_dir, zephyr_board, west_cmd, mod, build_config): project = tvm.micro.generate_project( str(TEMPLATE_PROJECT_DIR), mod, @@ -84,8 +84,7 @@ def _make_session(temp_dir, model, target, zephyr_board, west_cmd, mod, build_co "zephyr_board": zephyr_board, }, ) - if not build_config.get("skip_build"): - project.build() + project.build() project.flash() return tvm.micro.Session(project.transport()) @@ -102,11 +101,11 @@ def _make_add_sess(temp_dir, model, zephyr_board, west_cmd, build_config, dtype= # The same test code can be executed on both the QEMU simulation and on real hardware. @tvm.testing.requires_micro -def test_add_uint(temp_dir, platform, west_cmd, skip_build, tvm_debug): +def test_add_uint(temp_dir, platform, west_cmd, tvm_debug): """Test compiling the on-device runtime.""" model, zephyr_board = PLATFORMS[platform] - build_config = {"skip_build": skip_build, "debug": tvm_debug} + build_config = {"debug": tvm_debug} # NOTE: run test in a nested function so cPython will delete arrays before closing the session. def test_basic_add(sess): @@ -137,13 +136,13 @@ def has_fpu(zephyr_board): # The same test code can be executed on both the QEMU simulation and on real hardware. @tvm.testing.requires_micro -def test_add_float(temp_dir, platform, west_cmd, skip_build, tvm_debug): +def test_add_float(temp_dir, platform, west_cmd, tvm_debug): """Test compiling the on-device runtime.""" model, zephyr_board = PLATFORMS[platform] if not has_fpu(zephyr_board): pytest.skip(f"FPU not enabled for {platform}") - build_config = {"skip_build": skip_build, "debug": tvm_debug} + build_config = {"debug": tvm_debug} # NOTE: run test in a nested function so cPython will delete arrays before closing the session. def test_basic_add(sess): @@ -165,11 +164,11 @@ def test_basic_add(sess): @tvm.testing.requires_micro -def test_platform_timer(temp_dir, platform, west_cmd, skip_build, tvm_debug): +def test_platform_timer(temp_dir, platform, west_cmd, tvm_debug): """Test compiling the on-device runtime.""" model, zephyr_board = PLATFORMS[platform] - build_config = {"skip_build": skip_build, "debug": tvm_debug} + build_config = {"debug": tvm_debug} # NOTE: run test in a nested function so cPython will delete arrays before closing the session. def test_basic_add(sess): @@ -194,10 +193,10 @@ def test_basic_add(sess): @tvm.testing.requires_micro -def test_relay(temp_dir, platform, west_cmd, skip_build, tvm_debug): +def test_relay(temp_dir, platform, west_cmd, tvm_debug): """Testing a simple relay graph""" model, zephyr_board = PLATFORMS[platform] - build_config = {"skip_build": skip_build, "debug": tvm_debug} + build_config = {"debug": tvm_debug} shape = (10,) dtype = "int8" @@ -211,9 +210,7 @@ def test_relay(temp_dir, platform, west_cmd, skip_build, tvm_debug): with tvm.transform.PassContext(opt_level=3, config={"tir.disable_vectorize": True}): mod = tvm.relay.build(func, target=target) - with _make_session( - temp_dir, model, target, zephyr_board, west_cmd, mod, build_config - ) as session: + with _make_session(temp_dir, zephyr_board, west_cmd, mod, build_config) as session: graph_mod = tvm.micro.create_local_graph_executor( mod.get_graph_json(), session.get_system_lib(), session.device ) @@ -226,10 +223,10 @@ def test_relay(temp_dir, platform, west_cmd, skip_build, tvm_debug): @tvm.testing.requires_micro -def test_onnx(temp_dir, platform, west_cmd, skip_build, tvm_debug): +def test_onnx(temp_dir, platform, west_cmd, tvm_debug): """Testing a simple ONNX model.""" model, zephyr_board = PLATFORMS[platform] - build_config = {"skip_build": skip_build, "debug": tvm_debug} + build_config = {"debug": tvm_debug} # Load test images. this_dir = os.path.dirname(__file__) @@ -256,9 +253,7 @@ def test_onnx(temp_dir, platform, west_cmd, skip_build, tvm_debug): lowered = relay.build(relay_mod, target, params=params) graph = lowered.get_graph_json() - with _make_session( - temp_dir, model, target, zephyr_board, west_cmd, lowered, build_config - ) as session: + with _make_session(temp_dir, zephyr_board, west_cmd, lowered, build_config) as session: graph_mod = tvm.micro.create_local_graph_executor( graph, session.get_system_lib(), session.device ) @@ -285,9 +280,7 @@ def check_result( with tvm.transform.PassContext(opt_level=3, config={"tir.disable_vectorize": True}): mod = tvm.relay.build(relay_mod, target=target) - with _make_session( - temp_dir, model, target, zephyr_board, west_cmd, mod, build_config - ) as session: + with _make_session(temp_dir, zephyr_board, west_cmd, mod, build_config) as session: rt_mod = tvm.micro.create_local_graph_executor( mod.get_graph_json(), session.get_system_lib(), session.device ) @@ -307,10 +300,10 @@ def check_result( @tvm.testing.requires_micro -def test_byoc_microtvm(temp_dir, platform, west_cmd, skip_build, tvm_debug): +def test_byoc_microtvm(temp_dir, platform, west_cmd, tvm_debug): """This is a simple test case to check BYOC capabilities of microTVM""" model, zephyr_board = PLATFORMS[platform] - build_config = {"skip_build": skip_build, "debug": tvm_debug} + build_config = {"debug": tvm_debug} x = relay.var("x", shape=(10, 10)) w0 = relay.var("w0", shape=(10, 10)) w1 = relay.var("w1", shape=(10, 10)) @@ -387,10 +380,10 @@ def _make_add_sess_with_shape(temp_dir, model, zephyr_board, west_cmd, shape, bu ], ) @tvm.testing.requires_micro -def test_rpc_large_array(temp_dir, platform, west_cmd, skip_build, tvm_debug, shape): +def test_rpc_large_array(temp_dir, platform, west_cmd, tvm_debug, shape): """Test large RPC array transfer.""" model, zephyr_board = PLATFORMS[platform] - build_config = {"skip_build": skip_build, "debug": tvm_debug} + build_config = {"debug": tvm_debug} # NOTE: run test in a nested function so cPython will delete arrays before closing the session. def test_tensors(sess): diff --git a/tests/micro/zephyr/test_zephyr_aot.py b/tests/micro/zephyr/test_zephyr_aot.py index 4f1f646e8b1e..e143515cc442 100644 --- a/tests/micro/zephyr/test_zephyr_aot.py +++ b/tests/micro/zephyr/test_zephyr_aot.py @@ -45,9 +45,7 @@ PLATFORMS = conftest.PLATFORMS -def _build_project( - temp_dir, model, target, zephyr_board, west_cmd, mod, build_config, extra_files_tar=None -): +def _build_project(temp_dir, zephyr_board, west_cmd, mod, build_config, extra_files_tar=None): template_project_dir = ( pathlib.Path(__file__).parent / ".." @@ -67,7 +65,7 @@ def _build_project( "extra_files_tar": extra_files_tar, "project_type": "aot_demo", "west_cmd": west_cmd, - "verbose": 0, + "verbose": bool(build_config.get("debug")), "zephyr_board": zephyr_board, }, ) @@ -137,7 +135,7 @@ def _get_message(fd, expr: str, timeout_sec: int): @tvm.testing.requires_micro -def test_tflite(temp_dir, platform, west_cmd, skip_build, tvm_debug): +def test_tflite(temp_dir, platform, west_cmd, tvm_debug): """Testing a TFLite model.""" if platform not in ["host", "mps2_an521", "nrf5340dk", "stm32l4r5zi_nucleo", "zynq_mp_r5"]: @@ -146,11 +144,8 @@ def test_tflite(temp_dir, platform, west_cmd, skip_build, tvm_debug): model, zephyr_board = PLATFORMS[platform] input_shape = (1, 32, 32, 3) output_shape = (1, 10) - build_config = {"skip_build": skip_build, "debug": tvm_debug} + build_config = {"debug": tvm_debug} - this_dir = pathlib.Path(os.path.dirname(__file__)) - tvm_source_dir = this_dir / ".." / ".." / ".." - runtime_path = tvm_source_dir / "apps" / "microtvm" / "zephyr" / "aot_demo" model_url = "https://github.com/eembc/ulpmark-ml/raw/fc1499c7cc83681a02820d5ddf5d97fe75d4f663/base_models/ic01/ic01_fp32.tflite" model_path = download_testdata(model_url, "ic01_fp32.tflite", module="model") @@ -200,8 +195,6 @@ def test_tflite(temp_dir, platform, west_cmd, skip_build, tvm_debug): project, _ = _build_project( temp_dir, - model, - target, zephyr_board, west_cmd, lowered, @@ -225,13 +218,13 @@ def test_tflite(temp_dir, platform, west_cmd, skip_build, tvm_debug): @tvm.testing.requires_micro -def test_qemu_make_fail(temp_dir, platform, west_cmd, skip_build, tvm_debug): +def test_qemu_make_fail(temp_dir, platform, west_cmd, tvm_debug): """Testing QEMU make fail.""" if platform not in ["host", "mps2_an521"]: pytest.skip(msg="Only for QEMU targets.") model, zephyr_board = PLATFORMS[platform] - build_config = {"skip_build": skip_build, "debug": tvm_debug} + build_config = {"debug": tvm_debug} shape = (10,) dtype = "float32" @@ -260,8 +253,6 @@ def test_qemu_make_fail(temp_dir, platform, west_cmd, skip_build, tvm_debug): project, project_dir = _build_project( temp_dir, - model, - target, zephyr_board, west_cmd, lowered, diff --git a/tests/scripts/task_python_microtvm.sh b/tests/scripts/task_python_microtvm.sh index aa49d90eaa43..79b1072cc952 100755 --- a/tests/scripts/task_python_microtvm.sh +++ b/tests/scripts/task_python_microtvm.sh @@ -23,5 +23,5 @@ set -x # NOTE(areusch): Adding to diagnose flaky timeouts source tests/scripts/setup-pytest-env.sh make cython3 -run_pytest ctypes python-microtvm-zephyr tests/micro/zephyr --microtvm-platforms=host +run_pytest ctypes python-microtvm-zephyr tests/micro/zephyr --microtvm-platforms=qemu_x86 run_pytest ctypes python-microtvm-zephyr tests/micro/zephyr --microtvm-platforms=mps2_an521