[microTVM] Zephyr Test Refactor (#8713)

* refactor host to qemu

* remove unused variables

* remove skip-build arg

* fix microtvm test script

tests/micro/zephyr/conftest.py

@@ -26,9 +26,9 @@
 # The models that should pass this configuration. Maps a short, identifying platform string to
 # (model, zephyr_board).
 PLATFORMS = {
-    "host": ("host", "qemu_x86"),
-    "host_riscv32": ("host", "qemu_riscv32"),
-    "host_riscv64": ("host", "qemu_riscv64"),
+    "qemu_x86": ("host", "qemu_x86"),
+    "qemu_riscv32": ("host", "qemu_riscv32"),
+    "qemu_riscv64": ("host", "qemu_riscv64"),
     "mps2_an521": ("mps2_an521", "mps2_an521"),
     "nrf5340dk": ("nrf5340dk", "nrf5340dk_nrf5340_cpuapp"),
     "stm32f746xx_disco": ("stm32f746xx", "stm32f746g_disco"),
@@ -41,7 +41,7 @@
 def pytest_addoption(parser):
     parser.addoption(
         "--microtvm-platforms",
-        default="host",
+        default="qemu_x86",
         choices=PLATFORMS.keys(),
         help=(
             "Specify a comma-separated list of test models (i.e. as passed to tvm.target.micro()) "
@@ -51,11 +51,6 @@ def pytest_addoption(parser):
     parser.addoption(
         "--west-cmd", default="west", help="Path to `west` command for flashing device."
     )
-    parser.addoption(
-        "--skip-build",
-        action="store_true",
-        help="If set true, reuses build from the previous test run. Otherwise, build from the scratch.",
-    )
     parser.addoption(
         "--tvm-debug",
         action="store_true",
@@ -74,11 +69,6 @@ def west_cmd(request):
     return request.config.getoption("--west-cmd")
 
 
-@pytest.fixture
-def skip_build(request):
-    return request.config.getoption("--skip-build")
-
-
 @pytest.fixture
 def tvm_debug(request):
     return request.config.getoption("--tvm-debug")

tests/micro/zephyr/test_zephyr.py

@@ -57,7 +57,7 @@ def _make_sess_from_op(
     with tvm.transform.PassContext(opt_level=3, config={"tir.disable_vectorize": True}):
         mod = tvm.build(sched, arg_bufs, target=target, name=op_name)
 
-    return _make_session(temp_dir, model, target, zephyr_board, west_cmd, mod, build_config)
+    return _make_session(temp_dir, zephyr_board, west_cmd, mod, build_config)
 
 
 TEMPLATE_PROJECT_DIR = (
@@ -72,7 +72,7 @@ def _make_sess_from_op(
 ).resolve()
 
 
-def _make_session(temp_dir, model, target, zephyr_board, west_cmd, mod, build_config):
+def _make_session(temp_dir, zephyr_board, west_cmd, mod, build_config):
     project = tvm.micro.generate_project(
         str(TEMPLATE_PROJECT_DIR),
         mod,
@@ -84,8 +84,7 @@ def _make_session(temp_dir, model, target, zephyr_board, west_cmd, mod, build_co
             "zephyr_board": zephyr_board,
         },
     )
-    if not build_config.get("skip_build"):
-        project.build()
+    project.build()
     project.flash()
     return tvm.micro.Session(project.transport())
 
@@ -102,11 +101,11 @@ def _make_add_sess(temp_dir, model, zephyr_board, west_cmd, build_config, dtype=
 
 # The same test code can be executed on both the QEMU simulation and on real hardware.
 @tvm.testing.requires_micro
-def test_add_uint(temp_dir, platform, west_cmd, skip_build, tvm_debug):
+def test_add_uint(temp_dir, platform, west_cmd, tvm_debug):
     """Test compiling the on-device runtime."""
 
     model, zephyr_board = PLATFORMS[platform]
-    build_config = {"skip_build": skip_build, "debug": tvm_debug}
+    build_config = {"debug": tvm_debug}
 
     # NOTE: run test in a nested function so cPython will delete arrays before closing the session.
     def test_basic_add(sess):
@@ -137,13 +136,13 @@ def has_fpu(zephyr_board):
 
 # The same test code can be executed on both the QEMU simulation and on real hardware.
 @tvm.testing.requires_micro
-def test_add_float(temp_dir, platform, west_cmd, skip_build, tvm_debug):
+def test_add_float(temp_dir, platform, west_cmd, tvm_debug):
     """Test compiling the on-device runtime."""
     model, zephyr_board = PLATFORMS[platform]
     if not has_fpu(zephyr_board):
         pytest.skip(f"FPU not enabled for {platform}")
 
-    build_config = {"skip_build": skip_build, "debug": tvm_debug}
+    build_config = {"debug": tvm_debug}
 
     # NOTE: run test in a nested function so cPython will delete arrays before closing the session.
     def test_basic_add(sess):
@@ -165,11 +164,11 @@ def test_basic_add(sess):
 
 
 @tvm.testing.requires_micro
-def test_platform_timer(temp_dir, platform, west_cmd, skip_build, tvm_debug):
+def test_platform_timer(temp_dir, platform, west_cmd, tvm_debug):
     """Test compiling the on-device runtime."""
 
     model, zephyr_board = PLATFORMS[platform]
-    build_config = {"skip_build": skip_build, "debug": tvm_debug}
+    build_config = {"debug": tvm_debug}
 
     # NOTE: run test in a nested function so cPython will delete arrays before closing the session.
     def test_basic_add(sess):
@@ -194,10 +193,10 @@ def test_basic_add(sess):
 
 
 @tvm.testing.requires_micro
-def test_relay(temp_dir, platform, west_cmd, skip_build, tvm_debug):
+def test_relay(temp_dir, platform, west_cmd, tvm_debug):
     """Testing a simple relay graph"""
     model, zephyr_board = PLATFORMS[platform]
-    build_config = {"skip_build": skip_build, "debug": tvm_debug}
+    build_config = {"debug": tvm_debug}
     shape = (10,)
     dtype = "int8"
 
@@ -211,9 +210,7 @@ def test_relay(temp_dir, platform, west_cmd, skip_build, tvm_debug):
     with tvm.transform.PassContext(opt_level=3, config={"tir.disable_vectorize": True}):
         mod = tvm.relay.build(func, target=target)
 
-    with _make_session(
-        temp_dir, model, target, zephyr_board, west_cmd, mod, build_config
-    ) as session:
+    with _make_session(temp_dir, zephyr_board, west_cmd, mod, build_config) as session:
         graph_mod = tvm.micro.create_local_graph_executor(
             mod.get_graph_json(), session.get_system_lib(), session.device
         )
@@ -226,10 +223,10 @@ def test_relay(temp_dir, platform, west_cmd, skip_build, tvm_debug):
 
 
 @tvm.testing.requires_micro
-def test_onnx(temp_dir, platform, west_cmd, skip_build, tvm_debug):
+def test_onnx(temp_dir, platform, west_cmd, tvm_debug):
     """Testing a simple ONNX model."""
     model, zephyr_board = PLATFORMS[platform]
-    build_config = {"skip_build": skip_build, "debug": tvm_debug}
+    build_config = {"debug": tvm_debug}
 
     # Load test images.
     this_dir = os.path.dirname(__file__)
@@ -256,9 +253,7 @@ def test_onnx(temp_dir, platform, west_cmd, skip_build, tvm_debug):
         lowered = relay.build(relay_mod, target, params=params)
         graph = lowered.get_graph_json()
 
-    with _make_session(
-        temp_dir, model, target, zephyr_board, west_cmd, lowered, build_config
-    ) as session:
+    with _make_session(temp_dir, zephyr_board, west_cmd, lowered, build_config) as session:
         graph_mod = tvm.micro.create_local_graph_executor(
             graph, session.get_system_lib(), session.device
         )
@@ -285,9 +280,7 @@ def check_result(
     with tvm.transform.PassContext(opt_level=3, config={"tir.disable_vectorize": True}):
         mod = tvm.relay.build(relay_mod, target=target)
 
-    with _make_session(
-        temp_dir, model, target, zephyr_board, west_cmd, mod, build_config
-    ) as session:
+    with _make_session(temp_dir, zephyr_board, west_cmd, mod, build_config) as session:
         rt_mod = tvm.micro.create_local_graph_executor(
             mod.get_graph_json(), session.get_system_lib(), session.device
         )
@@ -307,10 +300,10 @@ def check_result(
 
 
 @tvm.testing.requires_micro
-def test_byoc_microtvm(temp_dir, platform, west_cmd, skip_build, tvm_debug):
+def test_byoc_microtvm(temp_dir, platform, west_cmd, tvm_debug):
     """This is a simple test case to check BYOC capabilities of microTVM"""
     model, zephyr_board = PLATFORMS[platform]
-    build_config = {"skip_build": skip_build, "debug": tvm_debug}
+    build_config = {"debug": tvm_debug}
     x = relay.var("x", shape=(10, 10))
     w0 = relay.var("w0", shape=(10, 10))
     w1 = relay.var("w1", shape=(10, 10))
@@ -387,10 +380,10 @@ def _make_add_sess_with_shape(temp_dir, model, zephyr_board, west_cmd, shape, bu
     ],
 )
 @tvm.testing.requires_micro
-def test_rpc_large_array(temp_dir, platform, west_cmd, skip_build, tvm_debug, shape):
+def test_rpc_large_array(temp_dir, platform, west_cmd, tvm_debug, shape):
     """Test large RPC array transfer."""
     model, zephyr_board = PLATFORMS[platform]
-    build_config = {"skip_build": skip_build, "debug": tvm_debug}
+    build_config = {"debug": tvm_debug}
 
     # NOTE: run test in a nested function so cPython will delete arrays before closing the session.
     def test_tensors(sess):

tests/micro/zephyr/test_zephyr_aot.py

@@ -45,9 +45,7 @@
 PLATFORMS = conftest.PLATFORMS
 
 
-def _build_project(
-    temp_dir, model, target, zephyr_board, west_cmd, mod, build_config, extra_files_tar=None
-):
+def _build_project(temp_dir, zephyr_board, west_cmd, mod, build_config, extra_files_tar=None):
     template_project_dir = (
         pathlib.Path(__file__).parent
         / ".."
@@ -67,7 +65,7 @@ def _build_project(
             "extra_files_tar": extra_files_tar,
             "project_type": "aot_demo",
             "west_cmd": west_cmd,
-            "verbose": 0,
+            "verbose": bool(build_config.get("debug")),
             "zephyr_board": zephyr_board,
         },
     )
@@ -137,7 +135,7 @@ def _get_message(fd, expr: str, timeout_sec: int):
 
 
 @tvm.testing.requires_micro
-def test_tflite(temp_dir, platform, west_cmd, skip_build, tvm_debug):
+def test_tflite(temp_dir, platform, west_cmd, tvm_debug):
     """Testing a TFLite model."""
 
     if platform not in ["host", "mps2_an521", "nrf5340dk", "stm32l4r5zi_nucleo", "zynq_mp_r5"]:
@@ -146,11 +144,8 @@ def test_tflite(temp_dir, platform, west_cmd, skip_build, tvm_debug):
     model, zephyr_board = PLATFORMS[platform]
     input_shape = (1, 32, 32, 3)
     output_shape = (1, 10)
-    build_config = {"skip_build": skip_build, "debug": tvm_debug}
+    build_config = {"debug": tvm_debug}
 
-    this_dir = pathlib.Path(os.path.dirname(__file__))
-    tvm_source_dir = this_dir / ".." / ".." / ".."
-    runtime_path = tvm_source_dir / "apps" / "microtvm" / "zephyr" / "aot_demo"
     model_url = "https://github.com/eembc/ulpmark-ml/raw/fc1499c7cc83681a02820d5ddf5d97fe75d4f663/base_models/ic01/ic01_fp32.tflite"
     model_path = download_testdata(model_url, "ic01_fp32.tflite", module="model")
 
@@ -200,8 +195,6 @@ def test_tflite(temp_dir, platform, west_cmd, skip_build, tvm_debug):
 
         project, _ = _build_project(
             temp_dir,
-            model,
-            target,
             zephyr_board,
             west_cmd,
             lowered,
@@ -225,13 +218,13 @@ def test_tflite(temp_dir, platform, west_cmd, skip_build, tvm_debug):
 
 
 @tvm.testing.requires_micro
-def test_qemu_make_fail(temp_dir, platform, west_cmd, skip_build, tvm_debug):
+def test_qemu_make_fail(temp_dir, platform, west_cmd, tvm_debug):
     """Testing QEMU make fail."""
     if platform not in ["host", "mps2_an521"]:
         pytest.skip(msg="Only for QEMU targets.")
 
     model, zephyr_board = PLATFORMS[platform]
-    build_config = {"skip_build": skip_build, "debug": tvm_debug}
+    build_config = {"debug": tvm_debug}
     shape = (10,)
     dtype = "float32"
 
@@ -260,8 +253,6 @@ def test_qemu_make_fail(temp_dir, platform, west_cmd, skip_build, tvm_debug):
 
         project, project_dir = _build_project(
             temp_dir,
-            model,
-            target,
             zephyr_board,
             west_cmd,
             lowered,

tests/scripts/task_python_microtvm.sh

@@ -23,5 +23,5 @@ set -x  # NOTE(areusch): Adding to diagnose flaky timeouts
 source tests/scripts/setup-pytest-env.sh
 
 make cython3
-run_pytest ctypes python-microtvm-zephyr tests/micro/zephyr --microtvm-platforms=host
+run_pytest ctypes python-microtvm-zephyr tests/micro/zephyr --microtvm-platforms=qemu_x86
 run_pytest ctypes python-microtvm-zephyr tests/micro/zephyr --microtvm-platforms=mps2_an521