From 12fd15c99c2dba512a6326a3b04d090985524d1d Mon Sep 17 00:00:00 2001
From: Andrei Litvin <andy314@gmail.com>
Date: Thu, 16 Dec 2021 11:39:30 -0500
Subject: [PATCH 01/22] Start adding a tests target to qemu

---
 .github/workflows/qemu.yaml     |  3 ---
 scripts/build/build/targets.py  |  2 ++
 scripts/build/builders/esp32.py | 40 +++++++++++++++++++++++++--------
 3 files changed, 33 insertions(+), 12 deletions(-)

diff --git a/.github/workflows/qemu.yaml b/.github/workflows/qemu.yaml
index 6f68a6409e7ea7..239db16b6fa2bd 100644
--- a/.github/workflows/qemu.yaml
+++ b/.github/workflows/qemu.yaml
@@ -54,9 +54,6 @@ jobs:
                   path: |
                    .environment/gn_out/.ninja_log
                    .environment/pigweed-venv/*.log
-            - name: Build example All Clusters App
-              timeout-minutes: 25
-              run: scripts/examples/esp_example.sh all-clusters-app
             - name: Build ESP32 QEMU and Run Tests
               timeout-minutes: 35
               run: scripts/tests/esp32_qemu_tests.sh /tmp/test_logs
diff --git a/scripts/build/build/targets.py b/scripts/build/build/targets.py
index 3dbaa16ea89e83..043fad9c5b1691 100644
--- a/scripts/build/build/targets.py
+++ b/scripts/build/build/targets.py
@@ -220,6 +220,8 @@ def Esp32Targets():
     yield devkitc.Extend('bridge', app=Esp32App.BRIDGE)
     yield devkitc.Extend('temperature-measurement', app=Esp32App.TEMPERATURE_MEASUREMENT)
 
+    yield esp32_target.Extend('qemu-tests', board=Esp32Board.QEMU, app=Esp32App.TESTS)
+
 
 def Efr32Targets():
     efr_target = Target('efr32', Efr32Builder)
diff --git a/scripts/build/builders/esp32.py b/scripts/build/builders/esp32.py
index f3f709e4e76e83..0f4b43c744ba02 100644
--- a/scripts/build/builders/esp32.py
+++ b/scripts/build/builders/esp32.py
@@ -25,6 +25,7 @@ class Esp32Board(Enum):
     DevKitC = auto()
     M5Stack = auto()
     C3DevKit = auto()
+    QEMU = auto()
 
 
 class Esp32App(Enum):
@@ -33,19 +34,22 @@ class Esp32App(Enum):
     SHELL = auto()
     BRIDGE = auto()
     TEMPERATURE_MEASUREMENT = auto()
+    TESTS = auto()
 
     @property
-    def ExampleName(self):
+    def ExamplePath(self):
         if self == Esp32App.ALL_CLUSTERS:
-            return 'all-clusters-app'
+            return 'examples/all-clusters-app'
         elif self == Esp32App.LOCK:
-            return 'lock-app'
+            return 'examples/lock-app'
         elif self == Esp32App.SHELL:
-            return 'shell'
+            return 'examples/shell'
         elif self == Esp32App.BRIDGE:
-            return 'bridge-app'
+            return 'examples/bridge-app'
         elif self == Esp32App.TEMPERATURE_MEASUREMENT:
-            return 'temperature-measurement-app'
+            return 'examples/temperature-measurement-app'
+        elif self == Esp32App.TESTS:
+            return 'src/test_driver'
         else:
             raise Exception('Unknown app type: %r' % self)
 
@@ -61,15 +65,30 @@ def AppNamePrefix(self):
             return 'chip-bridge-app'
         elif self == Esp32App.TEMPERATURE_MEASUREMENT:
             return 'chip-temperature-measurement-app'
+        elif self == Esp32App.TESTS:
+            return 'FIXME_FIXME_FIXME'
         else:
             raise Exception('Unknown app type: %r' % self)
 
+    @property
     def FlashBundleName(self):
         return self.AppNamePrefix + '.flashbundle.txt'
 
+    def IsCompatible(self, board:Esp32Board):
+        if board == Esp32Board.QEMU:
+            return self == Esp32App.TESTS
+        elif board == Esp32Board.M5Stack:
+            return self == Esp32App.ALL_CLUSTERS
+        elif board == Esp32Board.C3DevKit:
+            return self == Esp32App.ALL_CLUSTERS
+        else:
+            return board == Esp32Board.DevKitC
+
 
 def DefaultsFileName(board: Esp32Board, app: Esp32App, enable_rpcs: bool):
-    if app != Esp32App.ALL_CLUSTERS:
+    if app == Esp32App.TESTS:
+        return 'sdkconfig_qemu.defaults'
+    elif app != Esp32App.ALL_CLUSTERS:
         return 'sdkconfig.defaults'
 
     rpc = "_rpc" if enable_rpcs else ""
@@ -99,6 +118,9 @@ def __init__(self,
         self.enable_rpcs = enable_rpcs
         self.enable_ipv4 = enable_ipv4
 
+        if not app.IsCompatible(board):
+            raise Exception("Incompatible app/board combination: %r and %r", app, board)
+
     def _IdfEnvExecute(self, cmd, title=None):
         # Run activate.sh after export.sh to ensure using the chip environment.
         self._Execute(
@@ -107,7 +129,7 @@ def _IdfEnvExecute(self, cmd, title=None):
 
     @property
     def ExamplePath(self):
-        return os.path.join('examples', self.app.ExampleName, 'esp32')
+        return os.path.join(self.app.ExamplePath, 'esp32')
 
     def generate(self):
         if os.path.exists(os.path.join(self.output_dir, 'build.ninja')):
@@ -170,7 +192,7 @@ def build_outputs(self):
         }
 
     def flashbundle(self):
-        with open(os.path.join(self.output_dir, self.app.FlashBundleName()), 'r') as fp:
+        with open(os.path.join(self.output_dir, self.app.FlashBundleName), 'r') as fp:
             return {
                 l.strip(): os.path.join(self.output_dir, l.strip()) for l in fp.readlines() if l.strip()
             }

From 2efbcb5519cfa6bd3d208ffb9a400c78a5380391 Mon Sep 17 00:00:00 2001
From: Andrei Litvin <andy314@gmail.com>
Date: Thu, 16 Dec 2021 11:41:33 -0500
Subject: [PATCH 02/22] Fix extra compatibility test

---
 scripts/build/builders/esp32.py   | 2 +-
 scripts/tests/esp32_qemu_tests.sh | 3 +++
 scripts/tools/qemu_run_test.sh    | 6 ++++++
 3 files changed, 10 insertions(+), 1 deletion(-)

diff --git a/scripts/build/builders/esp32.py b/scripts/build/builders/esp32.py
index 0f4b43c744ba02..e05042161ce982 100644
--- a/scripts/build/builders/esp32.py
+++ b/scripts/build/builders/esp32.py
@@ -82,7 +82,7 @@ def IsCompatible(self, board:Esp32Board):
         elif board == Esp32Board.C3DevKit:
             return self == Esp32App.ALL_CLUSTERS
         else:
-            return board == Esp32Board.DevKitC
+            return (board == Esp32Board.DevKitC) and (self  != Esp32App.TESTS)
 
 
 def DefaultsFileName(board: Esp32Board, app: Esp32App, enable_rpcs: bool):
diff --git a/scripts/tests/esp32_qemu_tests.sh b/scripts/tests/esp32_qemu_tests.sh
index 942c9403962f61..d1fe165fd1ef6c 100755
--- a/scripts/tests/esp32_qemu_tests.sh
+++ b/scripts/tests/esp32_qemu_tests.sh
@@ -23,6 +23,7 @@
 set -e
 set -o pipefail
 set -x
+set -v
 
 here=$(cd "$(dirname "$0")" && pwd)
 chip_dir="$here"/../..
@@ -34,6 +35,7 @@ fi
 
 # shellcheck source=/dev/null
 source "$chip_dir"/src/test_driver/esp32/idf.sh
+echo "!!!!!!!!!!!!!!!!!!!!!!! QEMU SETUP"
 "$chip_dir"/src/test_driver/esp32/qemu_setup.sh
 
 if [ $? -ne 0 ]; then
@@ -42,6 +44,7 @@ if [ $? -ne 0 ]; then
 fi
 
 really_run_suite() {
+    echo "!!!!!!!!!!!!!!!!!!!!!!! ACTUAL RUNNING THE TEST (in an awkward way)"
     idf scripts/tools/qemu_run_test.sh src/test_driver/esp32/build/chip "$@"
 }
 
diff --git a/scripts/tools/qemu_run_test.sh b/scripts/tools/qemu_run_test.sh
index c2196bf7c657b7..7686de049257db 100755
--- a/scripts/tools/qemu_run_test.sh
+++ b/scripts/tools/qemu_run_test.sh
@@ -22,6 +22,7 @@
 
 set -e
 set -x
+set -v
 
 die() {
     echo "${me:?}: *** ERROR: " "${*}"
@@ -37,13 +38,18 @@ EXTRA_COMPILE_ARGUMENTS="$*" # generally -lFooHelperLibrary
 
 # shellcheck source=/dev/null
 source "$BUILD_DIR"/env.sh
+
+echo "@@@@@@@@@@@@@@@@@@@@@ BASH SCRIUPT FOR  EXECUTION"
 bash "$BUILD_DIR"/esp32_elf_builder.sh "$BUILD_DIR/lib/$QEMU_TEST_TARGET" "$EXTRA_COMPILE_ARGUMENTS"
 
 flash_image_file=$(mktemp)
 log_file=$(mktemp)
 trap '{ rm -f $flash_image_file $log_file; }' EXIT
 
+echo "@@@@@@@@@@@@@@@@@@@@@ IMAGE FLASHING"
 "$SRC_DIR"/scripts/tools/build_esp32_flash_image.sh "$BUILD_DIR"/chip-tests.bin "$flash_image_file"
+
+echo "@@@@@@@@@@@@@@@@@@@@@ QEMU TEST RUN"
 "$SRC_DIR"/scripts/tools/esp32_qemu_run.sh "$flash_image_file" | tee "$log_file"
 
 # If the logs contain failure message

From b08f5a750b11e1d65bef1e4f972750b7f4479bd7 Mon Sep 17 00:00:00 2001
From: Andrei Litvin <andy314@gmail.com>
Date: Thu, 16 Dec 2021 12:00:26 -0500
Subject: [PATCH 03/22] Prepare qemu builds for the test driver using cmake

---
 src/test_driver/esp32/CMakeLists.txt          | 47 +++++++++++++++++++
 src/test_driver/esp32/main/CMakeLists.txt     | 36 ++++++++++++++
 src/test_driver/esp32/sdkconfig_qemu.defaults |  4 +-
 3 files changed, 85 insertions(+), 2 deletions(-)
 create mode 100644 src/test_driver/esp32/CMakeLists.txt
 create mode 100644 src/test_driver/esp32/main/CMakeLists.txt

diff --git a/src/test_driver/esp32/CMakeLists.txt b/src/test_driver/esp32/CMakeLists.txt
new file mode 100644
index 00000000000000..f0b7c43a32404b
--- /dev/null
+++ b/src/test_driver/esp32/CMakeLists.txt
@@ -0,0 +1,47 @@
+#
+#    Copyright (c) 2021 Project CHIP Authors
+#    All rights reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+
+# The following lines of boilerplate have to be in your project's
+# CMakeLists in this exact order for cmake to work correctly
+cmake_minimum_required(VERSION 3.5)
+include($ENV{IDF_PATH}/tools/cmake/project.cmake)
+include(${CMAKE_CURRENT_LIST_DIR}/../../../examples/common/cmake/idf_flashing.cmake)
+
+set(EXTRA_COMPONENT_DIRS
+    "${CMAKE_CURRENT_LIST_DIR}/third_party/connectedhomeip/config/esp32/components"
+)
+
+project(test-driver)
+
+# C++17 is required for RPC build.
+idf_build_set_property(CXX_COMPILE_OPTIONS "-std=gnu++17;-Os;-DLWIP_IPV6_SCOPES=0;-DCHIP_HAVE_CONFIG_H" APPEND)
+idf_build_set_property(C_COMPILE_OPTIONS "-Os;-DLWIP_IPV6_SCOPES=0" APPEND)
+
+# FIXME: elfbuilder ?
+#    mkdir -p build/chip/
+#    echo "#!/bin/sh" > build/chip/esp32_elf_builder.sh
+#    echo set -e >> build/chip/esp32_elf_builder.sh
+#    echo set -x >> build/chip/esp32_elf_builder.sh
+#    echo $(CXX) $(CXXFLAGS) $(CPPFLAGS) -L$(PROJECT_PATH)/build/chip/lib -Wl,--whole-archive '$$1' -Wl,--no-whole-archive \
+#         -lnlunit-test $(LDFLAGS) -lnlfaultinjection '$$2' -o $(PROJECT_PATH)/build/chip-tests.elf -Wl,-Map=$(APP_MAP) >> build/chip/esp32_elf_builder.sh
+#    echo $(ESPTOOLPY) elf2image $(ESPTOOL_FLASH_OPTIONS) $(ESPTOOL_ELF2IMAGE_OPTIONS) \
+#         -o $(PROJECT_PATH)/build/chip/chip-tests.bin $(PROJECT_PATH)/build/chip-tests.elf >> build/chip/esp32_elf_builder.sh
+#    ln -sf $(PROJECT_PATH)/build/partitions.bin $(PROJECT_PATH)/build/chip/partitions.bin
+#    mkdir -p build/chip/bootloader
+#    ln -sf $(PROJECT_PATH)/build/bootloader/bootloader.bin $(PROJECT_PATH)/build/chip/bootloader.bin
+#    ln -sf $(PROJECT_PATH)/idf.sh $(PROJECT_PATH)/build/chip/env.sh
+
+flashing_script()
diff --git a/src/test_driver/esp32/main/CMakeLists.txt b/src/test_driver/esp32/main/CMakeLists.txt
new file mode 100644
index 00000000000000..09ca119db75e23
--- /dev/null
+++ b/src/test_driver/esp32/main/CMakeLists.txt
@@ -0,0 +1,36 @@
+#
+#    Copyright (c) 2021 Project CHIP Authors
+#    All rights reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+# (Uses default behaviour of compiling all source files in directory, adding 'include' to include path.)
+# The list of src and include dirs must be in sync with that in all-clusters-app/esp32/main/component.mk
+set(PRIV_INCLUDE_DIRS_LIST
+                      "${CMAKE_CURRENT_LIST_DIR}/include"
+)
+set(SRC_DIRS_LIST
+                      "${CMAKE_CURRENT_LIST_DIR}"
+)
+
+if (CONFIG_OPENTHREAD_ENABLED)
+    list(APPEND PRIV_REQUIRES_LIST openthread)
+endif()
+
+idf_component_register(PRIV_INCLUDE_DIRS ${PRIV_INCLUDE_DIRS_LIST}
+                       SRC_DIRS ${SRC_DIRS_LIST}
+                       PRIV_REQUIRES ${PRIV_REQUIRES_LIST})
+
+set_property(TARGET ${COMPONENT_LIB} PROPERTY CXX_STANDARD 17)
+target_compile_options(${COMPONENT_LIB} PRIVATE "-DLWIP_IPV6_SCOPES=0" "-DCHIP_HAVE_CONFIG_H")
diff --git a/src/test_driver/esp32/sdkconfig_qemu.defaults b/src/test_driver/esp32/sdkconfig_qemu.defaults
index 6509c5ba7d9f2e..2c921c3bb167d0 100644
--- a/src/test_driver/esp32/sdkconfig_qemu.defaults
+++ b/src/test_driver/esp32/sdkconfig_qemu.defaults
@@ -31,8 +31,8 @@ CONFIG_MBEDTLS_HARDWARE_MPI=n
 CONFIG_MBEDTLS_HARDWARE_SHA=n
 CONFIG_CHIP_TASK_STACK_SIZE=32768
 CONFIG_ESP_MAIN_TASK_STACK_SIZE=32768
-CONFIG_ESP32_PANIC_PRINT_REBOOT=y
-CONFIG_ESP32_PANIC_PRINT_HALT=n
+CONFIG_ESP_SYSTEM_PANIC_PRINT_REBOOT=y
+CONFIG_ESP_SYSTEM_PANIC_PRINT_HALT=n
 
 # enable BT
 CONFIG_BT_ENABLED=n

From 0bd82d98a83d307547fe1d0bc346bfd8c81997c8 Mon Sep 17 00:00:00 2001
From: Andrei Litvin <andy314@gmail.com>
Date: Thu, 16 Dec 2021 12:01:37 -0500
Subject: [PATCH 04/22] Ensure sdkconfig can be ignored for esp32 test driver:
 this file is re-written by the cmake build

---
 src/test_driver/esp32/.gitignore | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/src/test_driver/esp32/.gitignore b/src/test_driver/esp32/.gitignore
index 84c048a73cc2e5..a90ab6f1e57d5a 100644
--- a/src/test_driver/esp32/.gitignore
+++ b/src/test_driver/esp32/.gitignore
@@ -1 +1,3 @@
 /build/
+/sdkconfig
+/sdkconfig.old

From fc50e7cba6d06d2bb8b8d4cced89d814cf2bdcc7 Mon Sep 17 00:00:00 2001
From: Andrei Litvin <andy314@gmail.com>
Date: Thu, 16 Dec 2021 13:06:44 -0500
Subject: [PATCH 05/22] In theory we compile  and link at least one test

---
 src/test_driver/esp32/CMakeLists.txt      | 49 ++++++++++++++++++++++-
 src/test_driver/esp32/dummy.c             |  0
 src/test_driver/esp32/main/CMakeLists.txt |  6 +++
 3 files changed, 53 insertions(+), 2 deletions(-)
 create mode 100644 src/test_driver/esp32/dummy.c

diff --git a/src/test_driver/esp32/CMakeLists.txt b/src/test_driver/esp32/CMakeLists.txt
index f0b7c43a32404b..fcacf7ec09f3a4 100644
--- a/src/test_driver/esp32/CMakeLists.txt
+++ b/src/test_driver/esp32/CMakeLists.txt
@@ -18,7 +18,6 @@
 # CMakeLists in this exact order for cmake to work correctly
 cmake_minimum_required(VERSION 3.5)
 include($ENV{IDF_PATH}/tools/cmake/project.cmake)
-include(${CMAKE_CURRENT_LIST_DIR}/../../../examples/common/cmake/idf_flashing.cmake)
 
 set(EXTRA_COMPONENT_DIRS
     "${CMAKE_CURRENT_LIST_DIR}/third_party/connectedhomeip/config/esp32/components"
@@ -30,6 +29,53 @@ project(test-driver)
 idf_build_set_property(CXX_COMPILE_OPTIONS "-std=gnu++17;-Os;-DLWIP_IPV6_SCOPES=0;-DCHIP_HAVE_CONFIG_H" APPEND)
 idf_build_set_property(C_COMPILE_OPTIONS "-Os;-DLWIP_IPV6_SCOPES=0" APPEND)
 
+SET(VALID_TESTS
+    libASN1Tests.a
+)
+
+
+add_executable(testASN1Tests.elf dummy.c)
+
+target_link_directories(testASN1Tests.elf PUBLIC 
+    ${CMAKE_CURRENT_BINARY_DIR}/esp-idf/chip/lib
+    ${CMAKE_CURRENT_BINARY_DIR}/foobar
+)
+
+target_link_libraries(testASN1Tests.elf PUBLIC
+    idf::main 
+
+    -Wl,--whole-archive libASN1Tests.a -Wl,--no-whole-archive
+    nlunit-test
+    nlfaultinjection
+)
+idf_build_executable(testASN1Tests.elf)
+# 
+# add_executable(two.elf main_app.cpp)
+# target_link_libraries(two.elf chip)
+# idf_build_executable(two.elf)
+
+
+# TODO:
+#  - CXX depending on some lib*.a
+#  - multiple outputs/apps?
+
+#   # TODO: libAppTests depends on MessagingTestHelpers, which depends on
+#   # NetworkTestHelpers.  That sort of depends on InetTestHelpers or
+#   # equivalent (to provide gSystemLayer, gInet, InitNetwork(),
+#   # ShutdownNetwork()) but there's only a POSIX implementation of that
+#   # last, which does not compile on ESP32.  Need to figure out how to
+#   # make that work.  See comments below for the transport layer tests,
+#   # which have the same issue.
+#   # run_suite libAppTests.a -lMessagingTestHelpers -lNetworkTestHelpers
+#   
+#   run_suite libASN1Tests.a
+#   run_suite libBleLayerTests.a
+#   run_suite libCoreTests.a
+#   run_suite libInetLayerTests.a
+#   run_suite libRetransmitTests.a
+#   run_suite libSystemLayerTests.a
+#   run_suite libChipCryptoTests.a "-lChipCertTestVectors"
+
 # FIXME: elfbuilder ?
 #    mkdir -p build/chip/
 #    echo "#!/bin/sh" > build/chip/esp32_elf_builder.sh
@@ -44,4 +90,3 @@ idf_build_set_property(C_COMPILE_OPTIONS "-Os;-DLWIP_IPV6_SCOPES=0" APPEND)
 #    ln -sf $(PROJECT_PATH)/build/bootloader/bootloader.bin $(PROJECT_PATH)/build/chip/bootloader.bin
 #    ln -sf $(PROJECT_PATH)/idf.sh $(PROJECT_PATH)/build/chip/env.sh
 
-flashing_script()
diff --git a/src/test_driver/esp32/dummy.c b/src/test_driver/esp32/dummy.c
new file mode 100644
index 00000000000000..e69de29bb2d1d6
diff --git a/src/test_driver/esp32/main/CMakeLists.txt b/src/test_driver/esp32/main/CMakeLists.txt
index 09ca119db75e23..ded51eb78bee8c 100644
--- a/src/test_driver/esp32/main/CMakeLists.txt
+++ b/src/test_driver/esp32/main/CMakeLists.txt
@@ -20,10 +20,16 @@
 set(PRIV_INCLUDE_DIRS_LIST
                       "${CMAKE_CURRENT_LIST_DIR}/include"
 )
+
 set(SRC_DIRS_LIST
                       "${CMAKE_CURRENT_LIST_DIR}"
 )
 
+set(EXTRA_COMPONENT_DIRS
+    "${CMAKE_CURRENT_LIST_DIR}/../third_party/connectedhomeip/config/esp32/components"
+    "${IDF_PATH}/examples/common_components"
+)
+
 if (CONFIG_OPENTHREAD_ENABLED)
     list(APPEND PRIV_REQUIRES_LIST openthread)
 endif()

From 2e85bbc091920d75194658d9cfc592f9d877365e Mon Sep 17 00:00:00 2001
From: Andrei Litvin <andy314@gmail.com>
Date: Thu, 16 Dec 2021 13:15:18 -0500
Subject: [PATCH 06/22] Tests now build for a few sub-items

---
 src/test_driver/esp32/CMakeLists.txt | 36 ++++++++++++++++++----------
 1 file changed, 24 insertions(+), 12 deletions(-)

diff --git a/src/test_driver/esp32/CMakeLists.txt b/src/test_driver/esp32/CMakeLists.txt
index fcacf7ec09f3a4..68cf971cd6248f 100644
--- a/src/test_driver/esp32/CMakeLists.txt
+++ b/src/test_driver/esp32/CMakeLists.txt
@@ -31,24 +31,36 @@ idf_build_set_property(C_COMPILE_OPTIONS "-Os;-DLWIP_IPV6_SCOPES=0" APPEND)
 
 SET(VALID_TESTS
     libASN1Tests.a
+    # libBleLayerTests.a
+    libCoreTests.a
+    libInetLayerTests.a
+    libRetransmitTests.a
+    libSystemLayerTests.a
+    # TODO: how do I do this ?
+    # libChipCryptoTests.a "-lChipCertTestVectors"
 )
 
+foreach (TEST_LIB IN LISTS VALID_TESTS) 
+    string(REGEX REPLACE "lib(.*)\\.a" "\\1" NAME ${TEST_LIB})
 
-add_executable(testASN1Tests.elf dummy.c)
+    SET(ELF_NAME "test${NAME}.elf")
 
-target_link_directories(testASN1Tests.elf PUBLIC 
-    ${CMAKE_CURRENT_BINARY_DIR}/esp-idf/chip/lib
-    ${CMAKE_CURRENT_BINARY_DIR}/foobar
-)
+    add_executable(${ELF_NAME} dummy.c)
+
+    target_link_directories(${ELF_NAME} PUBLIC 
+        ${CMAKE_CURRENT_BINARY_DIR}/esp-idf/chip/lib
+    )
+
+    target_link_libraries(${ELF_NAME} PUBLIC
+        idf::main 
+        -Wl,--whole-archive ${TEST_LIB} -Wl,--no-whole-archive
+        nlunit-test
+        nlfaultinjection
+    )
+    idf_build_executable(${ELF_NAME})
+endforeach()
 
-target_link_libraries(testASN1Tests.elf PUBLIC
-    idf::main 
 
-    -Wl,--whole-archive libASN1Tests.a -Wl,--no-whole-archive
-    nlunit-test
-    nlfaultinjection
-)
-idf_build_executable(testASN1Tests.elf)
 # 
 # add_executable(two.elf main_app.cpp)
 # target_link_libraries(two.elf chip)

From c3f506dfb0e712adcbfb76831fe98fd821d1d843 Mon Sep 17 00:00:00 2001
From: Andrei Litvin <andy314@gmail.com>
Date: Thu, 16 Dec 2021 13:15:58 -0500
Subject: [PATCH 07/22] Tests now build for a few sub-items

---
 src/test_driver/esp32/CMakeLists.txt | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/test_driver/esp32/CMakeLists.txt b/src/test_driver/esp32/CMakeLists.txt
index 68cf971cd6248f..0c6141bb4d0fe7 100644
--- a/src/test_driver/esp32/CMakeLists.txt
+++ b/src/test_driver/esp32/CMakeLists.txt
@@ -30,14 +30,13 @@ idf_build_set_property(CXX_COMPILE_OPTIONS "-std=gnu++17;-Os;-DLWIP_IPV6_SCOPES=
 idf_build_set_property(C_COMPILE_OPTIONS "-Os;-DLWIP_IPV6_SCOPES=0" APPEND)
 
 SET(VALID_TESTS
-    libASN1Tests.a
     # libBleLayerTests.a
+    libASN1Tests.a
     libCoreTests.a
     libInetLayerTests.a
     libRetransmitTests.a
     libSystemLayerTests.a
-    # TODO: how do I do this ?
-    # libChipCryptoTests.a "-lChipCertTestVectors"
+    libChipCryptoTests.a # TODO: "-lChipCertTestVectors"
 )
 
 foreach (TEST_LIB IN LISTS VALID_TESTS) 
@@ -54,6 +53,7 @@ foreach (TEST_LIB IN LISTS VALID_TESTS)
     target_link_libraries(${ELF_NAME} PUBLIC
         idf::main 
         -Wl,--whole-archive ${TEST_LIB} -Wl,--no-whole-archive
+        ChipCertTestVectors # TODO: only useful for CryptoTests
         nlunit-test
         nlfaultinjection
     )

From d6927f7fbd9f83c2c80f79d486cc64f06d34e7bd Mon Sep 17 00:00:00 2001
From: Andrei Litvin <andy314@gmail.com>
Date: Thu, 16 Dec 2021 14:26:01 -0500
Subject: [PATCH 08/22] move the cmake logic for esp tests into a separate
 cmakefile

---
 src/test_driver/esp32/CMakeLists.txt          | 73 +++++--------------
 .../esp32/cmake/esp32_unit_tests.cmake        | 33 +++++++++
 2 files changed, 52 insertions(+), 54 deletions(-)
 create mode 100644 src/test_driver/esp32/cmake/esp32_unit_tests.cmake

diff --git a/src/test_driver/esp32/CMakeLists.txt b/src/test_driver/esp32/CMakeLists.txt
index 0c6141bb4d0fe7..e8f55cb93b2898 100644
--- a/src/test_driver/esp32/CMakeLists.txt
+++ b/src/test_driver/esp32/CMakeLists.txt
@@ -19,6 +19,8 @@
 cmake_minimum_required(VERSION 3.5)
 include($ENV{IDF_PATH}/tools/cmake/project.cmake)
 
+include(${CMAKE_CURRENT_SOURCE_DIR}/cmake/esp32_unit_tests.cmake)
+
 set(EXTRA_COMPONENT_DIRS
     "${CMAKE_CURRENT_LIST_DIR}/third_party/connectedhomeip/config/esp32/components"
 )
@@ -29,64 +31,27 @@ project(test-driver)
 idf_build_set_property(CXX_COMPILE_OPTIONS "-std=gnu++17;-Os;-DLWIP_IPV6_SCOPES=0;-DCHIP_HAVE_CONFIG_H" APPEND)
 idf_build_set_property(C_COMPILE_OPTIONS "-Os;-DLWIP_IPV6_SCOPES=0" APPEND)
 
-SET(VALID_TESTS
-    # libBleLayerTests.a
-    libASN1Tests.a
-    libCoreTests.a
-    libInetLayerTests.a
-    libRetransmitTests.a
-    libSystemLayerTests.a
-    libChipCryptoTests.a # TODO: "-lChipCertTestVectors"
-)
-
-foreach (TEST_LIB IN LISTS VALID_TESTS) 
-    string(REGEX REPLACE "lib(.*)\\.a" "\\1" NAME ${TEST_LIB})
-
-    SET(ELF_NAME "test${NAME}.elf")
-
-    add_executable(${ELF_NAME} dummy.c)
-
-    target_link_directories(${ELF_NAME} PUBLIC 
-        ${CMAKE_CURRENT_BINARY_DIR}/esp-idf/chip/lib
-    )
-
-    target_link_libraries(${ELF_NAME} PUBLIC
-        idf::main 
-        -Wl,--whole-archive ${TEST_LIB} -Wl,--no-whole-archive
-        ChipCertTestVectors # TODO: only useful for CryptoTests
-        nlunit-test
-        nlfaultinjection
-    )
-    idf_build_executable(${ELF_NAME})
-endforeach()
-
-
+# TODO: libAppTests depends on MessagingTestHelpers, which depends on
+# NetworkTestHelpers.  That sort of depends on InetTestHelpers or
+# equivalent (to provide gSystemLayer, gInet, InitNetwork(),
+# ShutdownNetwork()) but there's only a POSIX implementation of that
+# last, which does not compile on ESP32.  Need to figure out how to
+# make that work.  See comments below for the transport layer tests,
+# which have the same issue.
+#
+# libAppTests.a -lMessagingTestHelpers -lNetworkTestHelpers
 # 
-# add_executable(two.elf main_app.cpp)
-# target_link_libraries(two.elf chip)
-# idf_build_executable(two.elf)
+# TODO: ble tests do not compile using CMake (library is not auto-built)
+# libBleLayerTests.a
 
+esp32_unit_test(NAME testASN1 LIBRARY ASN1Tests)
+esp32_unit_test(NAME testChipCrypto LIBRARY ChipCryptoTests EXTRA_LIBRARIES -lChipCertTestVectors)
+esp32_unit_test(NAME testCore LIBRARY CoreTests)
+esp32_unit_test(NAME testInetLayer LIBRARY InetLayerTests)
+esp32_unit_test(NAME testRetransmit LIBRARY RetransmitTests)
+esp32_unit_test(NAME testSystemLayer LIBRARY SystemLayerTests)
 
-# TODO:
-#  - CXX depending on some lib*.a
-#  - multiple outputs/apps?
 
-#   # TODO: libAppTests depends on MessagingTestHelpers, which depends on
-#   # NetworkTestHelpers.  That sort of depends on InetTestHelpers or
-#   # equivalent (to provide gSystemLayer, gInet, InitNetwork(),
-#   # ShutdownNetwork()) but there's only a POSIX implementation of that
-#   # last, which does not compile on ESP32.  Need to figure out how to
-#   # make that work.  See comments below for the transport layer tests,
-#   # which have the same issue.
-#   # run_suite libAppTests.a -lMessagingTestHelpers -lNetworkTestHelpers
-#   
-#   run_suite libASN1Tests.a
-#   run_suite libBleLayerTests.a
-#   run_suite libCoreTests.a
-#   run_suite libInetLayerTests.a
-#   run_suite libRetransmitTests.a
-#   run_suite libSystemLayerTests.a
-#   run_suite libChipCryptoTests.a "-lChipCertTestVectors"
 
 # FIXME: elfbuilder ?
 #    mkdir -p build/chip/
diff --git a/src/test_driver/esp32/cmake/esp32_unit_tests.cmake b/src/test_driver/esp32/cmake/esp32_unit_tests.cmake
new file mode 100644
index 00000000000000..5dfa0d5eb069f5
--- /dev/null
+++ b/src/test_driver/esp32/cmake/esp32_unit_tests.cmake
@@ -0,0 +1,33 @@
+
+# Defines a stand-alone elf unit test that can be executed in QEMU
+#
+# Parameters:
+#    NAME    - the elf file name
+#    LIBRARY - the library that contains the registered unit tests. 
+#              for libASN1Tests.a use ASN1Tests as name.
+#    EXTRA_LIBRAIRES - what else to add to link libraries, generally dependencies
+#                      of $LIBRARY   
+macro(esp32_unit_test)
+    cmake_parse_arguments(
+        UNIT_TEST
+          ""                 # options
+          "NAME;LIBRARY"     # one value arguments
+          "EXTRA_LIBRARIES"  # multi value arguments
+          ${ARGN}
+    )
+
+    add_executable(${UNIT_TEST_NAME} dummy.c)
+
+    target_link_directories(${UNIT_TEST_NAME} PUBLIC 
+        ${CMAKE_CURRENT_BINARY_DIR}/esp-idf/chip/lib
+    )
+
+    target_link_libraries(${UNIT_TEST_NAME} PUBLIC
+        idf::main 
+        -Wl,--whole-archive ${UNIT_TEST_LIBRARY} -Wl,--no-whole-archive
+        ${UNIT_TEST_EXTRA_LIBRARIES}
+        nlunit-test
+        nlfaultinjection
+    )
+    idf_build_executable(${UNIT_TEST_NAME})
+endmacro()
\ No newline at end of file

From 5993243ff025383ff4beefe76de4c8282790f999 Mon Sep 17 00:00:00 2001
From: Andrei Litvin <andy314@gmail.com>
Date: Thu, 16 Dec 2021 14:52:11 -0500
Subject: [PATCH 09/22] bin is also built by the cmake system now

---
 src/test_driver/esp32/CMakeLists.txt          | 17 -------
 .../esp32/cmake/esp32_unit_tests.cmake        | 45 ++++++++++++++++++-
 2 files changed, 44 insertions(+), 18 deletions(-)

diff --git a/src/test_driver/esp32/CMakeLists.txt b/src/test_driver/esp32/CMakeLists.txt
index e8f55cb93b2898..bd8756dca7553b 100644
--- a/src/test_driver/esp32/CMakeLists.txt
+++ b/src/test_driver/esp32/CMakeLists.txt
@@ -50,20 +50,3 @@ esp32_unit_test(NAME testCore LIBRARY CoreTests)
 esp32_unit_test(NAME testInetLayer LIBRARY InetLayerTests)
 esp32_unit_test(NAME testRetransmit LIBRARY RetransmitTests)
 esp32_unit_test(NAME testSystemLayer LIBRARY SystemLayerTests)
-
-
-
-# FIXME: elfbuilder ?
-#    mkdir -p build/chip/
-#    echo "#!/bin/sh" > build/chip/esp32_elf_builder.sh
-#    echo set -e >> build/chip/esp32_elf_builder.sh
-#    echo set -x >> build/chip/esp32_elf_builder.sh
-#    echo $(CXX) $(CXXFLAGS) $(CPPFLAGS) -L$(PROJECT_PATH)/build/chip/lib -Wl,--whole-archive '$$1' -Wl,--no-whole-archive \
-#         -lnlunit-test $(LDFLAGS) -lnlfaultinjection '$$2' -o $(PROJECT_PATH)/build/chip-tests.elf -Wl,-Map=$(APP_MAP) >> build/chip/esp32_elf_builder.sh
-#    echo $(ESPTOOLPY) elf2image $(ESPTOOL_FLASH_OPTIONS) $(ESPTOOL_ELF2IMAGE_OPTIONS) \
-#         -o $(PROJECT_PATH)/build/chip/chip-tests.bin $(PROJECT_PATH)/build/chip-tests.elf >> build/chip/esp32_elf_builder.sh
-#    ln -sf $(PROJECT_PATH)/build/partitions.bin $(PROJECT_PATH)/build/chip/partitions.bin
-#    mkdir -p build/chip/bootloader
-#    ln -sf $(PROJECT_PATH)/build/bootloader/bootloader.bin $(PROJECT_PATH)/build/chip/bootloader.bin
-#    ln -sf $(PROJECT_PATH)/idf.sh $(PROJECT_PATH)/build/chip/env.sh
-
diff --git a/src/test_driver/esp32/cmake/esp32_unit_tests.cmake b/src/test_driver/esp32/cmake/esp32_unit_tests.cmake
index 5dfa0d5eb069f5..7c0c1f4e65aa3c 100644
--- a/src/test_driver/esp32/cmake/esp32_unit_tests.cmake
+++ b/src/test_driver/esp32/cmake/esp32_unit_tests.cmake
@@ -7,6 +7,11 @@
 #              for libASN1Tests.a use ASN1Tests as name.
 #    EXTRA_LIBRAIRES - what else to add to link libraries, generally dependencies
 #                      of $LIBRARY   
+#
+# TODO: several paths are hard-coded here and could use some updates:
+#      - always links to idf::main
+#      - assumes esp-idf/chip/lib is where the built libraries reside
+#      - assumes a "dummy.c" source exists to be able to "add_executable"
 macro(esp32_unit_test)
     cmake_parse_arguments(
         UNIT_TEST
@@ -16,6 +21,8 @@ macro(esp32_unit_test)
           ${ARGN}
     )
 
+    ######################## Elf  binary #######################
+
     add_executable(${UNIT_TEST_NAME} dummy.c)
 
     target_link_directories(${UNIT_TEST_NAME} PUBLIC 
@@ -30,4 +37,40 @@ macro(esp32_unit_test)
         nlfaultinjection
     )
     idf_build_executable(${UNIT_TEST_NAME})
-endmacro()
\ No newline at end of file
+
+    ######################## flashable image #######################
+
+    # This is very hacky: ESP build system will generate this, but only for the FIRST
+    # executable that is used
+
+    add_custom_command(
+      OUTPUT "${CMAKE_CURRENT_BINARY_DIR}/${UNIT_TEST_NAME}.bin_timestamp"
+      COMMAND ${ESPTOOLPY} elf2image ${ESPTOOLPY_FLASH_OPTIONS} ${esptool_elf2image_args}
+          -o "${CMAKE_CURRENT_BINARY_DIR}/${UNIT_TEST_NAME}.bin" "${UNIT_TEST_NAME}"
+      COMMAND ${CMAKE_COMMAND} -E echo "Generated ${UNIT_TEST_NAME}.bin"
+      COMMAND ${CMAKE_COMMAND} -E md5sum "${CMAKE_CURRENT_BINARY_DIR}/${UNIT_TEST_NAME}" > "${CMAKE_CURRENT_BINARY_DIR}/${UNIT_TEST_NAME}.bin_timestamp"
+      DEPENDS ${UNIT_TEST_NAME}
+      VERBATIM                                                                                                                                                  
+      WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}
+      COMMENT "Generating binary image from ${UNIT_TEST_NAME}"
+    )                                                                                                                                                         
+    add_custom_target(gen_project_binary_${UNIT_TEST_NAME} ALL DEPENDS "${build_dir}/${UNIT_TEST_NAME}.bin_timestamp")
+
+endmacro()
+
+
+# FIXME: elfbuilder ?
+#    mkdir -p build/chip/
+#    echo "#!/bin/sh" > build/chip/esp32_elf_builder.sh
+#    echo set -e >> build/chip/esp32_elf_builder.sh
+#    echo set -x >> build/chip/esp32_elf_builder.sh
+#    echo $(CXX) $(CXXFLAGS) $(CPPFLAGS) -L$(PROJECT_PATH)/build/chip/lib -Wl,--whole-archive '$$1' -Wl,--no-whole-archive \
+#         -lnlunit-test $(LDFLAGS) -lnlfaultinjection '$$2' -o $(PROJECT_PATH)/build/chip-tests.elf -Wl,-Map=$(APP_MAP) >> build/chip/esp32_elf_builder.sh
+
+#    echo $(ESPTOOLPY) elf2image $(ESPTOOL_FLASH_OPTIONS) $(ESPTOOL_ELF2IMAGE_OPTIONS) \
+#         -o $(PROJECT_PATH)/build/chip/chip-tests.bin $(PROJECT_PATH)/build/chip-tests.elf >> build/chip/esp32_elf_builder.sh
+#    ln -sf $(PROJECT_PATH)/build/partitions.bin $(PROJECT_PATH)/build/chip/partitions.bin
+#    mkdir -p build/chip/bootloader
+#    ln -sf $(PROJECT_PATH)/build/bootloader/bootloader.bin $(PROJECT_PATH)/build/chip/bootloader.bin
+#    ln -sf $(PROJECT_PATH)/idf.sh $(PROJECT_PATH)/build/chip/env.sh
+

From c96c9af1b8f663df5c917778d8f6a1aa2db41765 Mon Sep 17 00:00:00 2001
From: Andrei Litvin <andy314@gmail.com>
Date: Thu, 16 Dec 2021 15:24:56 -0500
Subject: [PATCH 10/22] Start being able to run test - for now remove the qemu
 sh scripts

---
 scripts/tests/esp32_qemu_tests.sh             | 94 -------------------
 scripts/tools/esp32_qemu_run.sh               | 74 ---------------
 scripts/tools/qemu_run_test.sh                | 65 -------------
 src/crypto/tests/qemu_crypto_tests.sh         |  1 -
 src/inet/tests/qemu_inet_tests.sh             |  1 -
 src/system/tests/qemu_system_tests.sh         |  1 -
 src/test_driver/esp32/CMakeLists.txt          |  5 +
 .../esp32/cmake/esp32_unit_tests.cmake        | 46 ++++++---
 src/test_driver/esp32/qemu_setup.sh           | 40 --------
 src/transport/tests/qemu_transport_tests.sh   |  1 -
 10 files changed, 36 insertions(+), 292 deletions(-)
 delete mode 100755 scripts/tests/esp32_qemu_tests.sh
 delete mode 100755 scripts/tools/esp32_qemu_run.sh
 delete mode 100755 scripts/tools/qemu_run_test.sh
 delete mode 120000 src/crypto/tests/qemu_crypto_tests.sh
 delete mode 120000 src/inet/tests/qemu_inet_tests.sh
 delete mode 120000 src/system/tests/qemu_system_tests.sh
 delete mode 100755 src/test_driver/esp32/qemu_setup.sh
 delete mode 120000 src/transport/tests/qemu_transport_tests.sh

diff --git a/scripts/tests/esp32_qemu_tests.sh b/scripts/tests/esp32_qemu_tests.sh
deleted file mode 100755
index d1fe165fd1ef6c..00000000000000
--- a/scripts/tests/esp32_qemu_tests.sh
+++ /dev/null
@@ -1,94 +0,0 @@
-#!/usr/bin/env bash
-#
-#
-#    Copyright (c) 2020 Project CHIP Authors
-#
-#    Licensed under the Apache License, Version 2.0 (the "License");
-#    you may not use this file except in compliance with the License.
-#    You may obtain a copy of the License at
-#
-#        http://www.apache.org/licenses/LICENSE-2.0
-#
-#    Unless required by applicable law or agreed to in writing, software
-#    distributed under the License is distributed on an "AS IS" BASIS,
-#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#    See the License for the specific language governing permissions and
-#    limitations under the License.
-#
-#
-#    Description:
-#      This is scripts builds ESP32 QEMU, and runs CHIP unit tests using it.
-#
-
-set -e
-set -o pipefail
-set -x
-set -v
-
-here=$(cd "$(dirname "$0")" && pwd)
-chip_dir="$here"/../..
-
-if [[ -n "$1" ]]; then
-    log_dir=$1
-    shift
-fi
-
-# shellcheck source=/dev/null
-source "$chip_dir"/src/test_driver/esp32/idf.sh
-echo "!!!!!!!!!!!!!!!!!!!!!!! QEMU SETUP"
-"$chip_dir"/src/test_driver/esp32/qemu_setup.sh
-
-if [ $? -ne 0 ]; then
-    echo "Setup failure"
-    exit 1
-fi
-
-really_run_suite() {
-    echo "!!!!!!!!!!!!!!!!!!!!!!! ACTUAL RUNNING THE TEST (in an awkward way)"
-    idf scripts/tools/qemu_run_test.sh src/test_driver/esp32/build/chip "$@"
-}
-
-run_suite() {
-    if [[ -d "${log_dir}" ]]; then
-        suite=${1%.a}
-        suite=${suite#lib}
-        really_run_suite "$@" 2>&1 | tee "$log_dir/$suite.log"
-    else
-        really_run_suite "$@"
-    fi
-}
-
-# Currently only crypto, inet, and system tests are configured to run on QEMU.
-# The specific qualifiers will be removed, once all CHIP unit tests are
-# updated to run on QEMU.
-SUITES=(
-)
-
-# TODO: libAppTests depends on MessagingTestHelpers, which depends on
-# NetworkTestHelpers.  That sort of depends on InetTestHelpers or
-# equivalent (to provide gSystemLayer, gInet, InitNetwork(),
-# ShutdownNetwork()) but there's only a POSIX implementation of that
-# last, which does not compile on ESP32.  Need to figure out how to
-# make that work.  See comments below for the transport layer tests,
-# which have the same issue.
-# run_suite libAppTests.a -lMessagingTestHelpers -lNetworkTestHelpers
-
-run_suite libASN1Tests.a
-run_suite libBleLayerTests.a
-run_suite libCoreTests.a
-run_suite libInetLayerTests.a
-run_suite libRetransmitTests.a
-run_suite libSystemLayerTests.a
-run_suite libChipCryptoTests.a "-lChipCertTestVectors"
-
-# TODO: Transport layer tests do not link:
-#    - getpid undefined
-#    - ArgParser for IPAddresses are not linked in
-#    - std::__throw_bad_alloc() linker errors
-# run_suite libRawTransportTests.a "-lNetworkTestHelpers -lInetTestHelpers"
-
-# TODO: Transport layer tests do not link:
-#    - getpid undefined
-#    - ArgParser for IPAddresses are not linked in
-#    - std::__throw_bad_alloc() linker errors
-# run_suite libTransportLayerTests.a "-lNetworkTestHelpers -lInetTestHelpers"
diff --git a/scripts/tools/esp32_qemu_run.sh b/scripts/tools/esp32_qemu_run.sh
deleted file mode 100755
index fb4e1e092a99d0..00000000000000
--- a/scripts/tools/esp32_qemu_run.sh
+++ /dev/null
@@ -1,74 +0,0 @@
-#!/usr/bin/env bash
-#
-#
-#    Copyright (c) 2020 Project CHIP Authors
-#
-#    Licensed under the Apache License, Version 2.0 (the "License");
-#    you may not use this file except in compliance with the License.
-#    You may obtain a copy of the License at
-#
-#        http://www.apache.org/licenses/LICENSE-2.0
-#
-#    Unless required by applicable law or agreed to in writing, software
-#    distributed under the License is distributed on an "AS IS" BASIS,
-#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#    See the License for the specific language governing permissions and
-#    limitations under the License.
-#
-
-#
-#    Description:
-#      This is a utility script that runs ESP32 QEMU using the given
-#      application image.
-#
-
-usage() {
-    exitcode=0
-    if [[ -n "$1" ]]; then
-        exitcode=1
-        echo "*** Error: $*"
-    fi
-    echo "Usage: $0 <flash image file>"
-    exit "$exitcode"
-}
-
-me=$(basename "$0")
-die() {
-    echo "$me: *** ERROR: " "${*}"
-    exit 1
-}
-
-realpath() {
-    path=$1 # input
-
-    [[ -z $path ]] && return 0
-
-    # trim trailing slashes
-    while [[ ${#path} -gt 1 && $path = */ ]]; do
-        path=${path%/}
-    done
-
-    # if we're at root we're done
-    if [[ $path = / ]]; then
-        echo "$path"
-        return 0
-    fi
-
-    [[ $path != /* ]] && path=$PWD/$path
-
-    if [[ -d $path ]]; then
-        (cd "$path" && pwd)
-    else
-        echo "$(realpath "${path%/*}")/${path##*/}"
-    fi
-}
-
-[[ $# -eq 1 ]] || usage "Incorrect number of arguments"
-
-[[ -n $QEMU_ESP32 ]] || die "Environment variable QEMU_ESP32 is undefined."
-
-flash_image=$(realpath "$1")
-
-[[ -r $flash_image ]] || usage "Could not read file $flash_image"
-
-"$QEMU_ESP32" -nographic -machine esp32 -drive file="$flash_image",if=mtd,format=raw -no-reboot
diff --git a/scripts/tools/qemu_run_test.sh b/scripts/tools/qemu_run_test.sh
deleted file mode 100755
index 7686de049257db..00000000000000
--- a/scripts/tools/qemu_run_test.sh
+++ /dev/null
@@ -1,65 +0,0 @@
-#!/usr/bin/env bash
-#
-#
-#    Copyright (c) 2020 Project CHIP Authors
-#
-#    Licensed under the Apache License, Version 2.0 (the "License");
-#    you may not use this file except in compliance with the License.
-#    You may obtain a copy of the License at
-#
-#        http://www.apache.org/licenses/LICENSE-2.0
-#
-#    Unless required by applicable law or agreed to in writing, software
-#    distributed under the License is distributed on an "AS IS" BASIS,
-#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#    See the License for the specific language governing permissions and
-#    limitations under the License.
-#
-#
-#    Description:
-#      This is a utility script that runs CHIP tests using ESP32 QEMU.
-#
-
-set -e
-set -x
-set -v
-
-die() {
-    echo "${me:?}: *** ERROR: " "${*}"
-    exit 1
-}
-
-SRC_DIR="$(dirname "$0")/../.."
-BUILD_DIR="$1"
-shift
-QEMU_TEST_TARGET="$1"
-shift
-EXTRA_COMPILE_ARGUMENTS="$*" # generally -lFooHelperLibrary
-
-# shellcheck source=/dev/null
-source "$BUILD_DIR"/env.sh
-
-echo "@@@@@@@@@@@@@@@@@@@@@ BASH SCRIUPT FOR  EXECUTION"
-bash "$BUILD_DIR"/esp32_elf_builder.sh "$BUILD_DIR/lib/$QEMU_TEST_TARGET" "$EXTRA_COMPILE_ARGUMENTS"
-
-flash_image_file=$(mktemp)
-log_file=$(mktemp)
-trap '{ rm -f $flash_image_file $log_file; }' EXIT
-
-echo "@@@@@@@@@@@@@@@@@@@@@ IMAGE FLASHING"
-"$SRC_DIR"/scripts/tools/build_esp32_flash_image.sh "$BUILD_DIR"/chip-tests.bin "$flash_image_file"
-
-echo "@@@@@@@@@@@@@@@@@@@@@ QEMU TEST RUN"
-"$SRC_DIR"/scripts/tools/esp32_qemu_run.sh "$flash_image_file" | tee "$log_file"
-
-# If the logs contain failure message
-if grep -F "] : FAILED" "$log_file"; then
-    die 'Some tests failed.'
-fi
-
-# If the logs do not contain final success status
-if grep -F "CHIP-tests: CHIP test status: 0" "$log_file"; then
-    echo "$me: All tests passed"
-else
-    die 'Tests did not run to completion.'
-fi
diff --git a/src/crypto/tests/qemu_crypto_tests.sh b/src/crypto/tests/qemu_crypto_tests.sh
deleted file mode 120000
index 8fbe617c97a267..00000000000000
--- a/src/crypto/tests/qemu_crypto_tests.sh
+++ /dev/null
@@ -1 +0,0 @@
-../../../scripts/tools/qemu_run_test.sh
\ No newline at end of file
diff --git a/src/inet/tests/qemu_inet_tests.sh b/src/inet/tests/qemu_inet_tests.sh
deleted file mode 120000
index 8fbe617c97a267..00000000000000
--- a/src/inet/tests/qemu_inet_tests.sh
+++ /dev/null
@@ -1 +0,0 @@
-../../../scripts/tools/qemu_run_test.sh
\ No newline at end of file
diff --git a/src/system/tests/qemu_system_tests.sh b/src/system/tests/qemu_system_tests.sh
deleted file mode 120000
index 8fbe617c97a267..00000000000000
--- a/src/system/tests/qemu_system_tests.sh
+++ /dev/null
@@ -1 +0,0 @@
-../../../scripts/tools/qemu_run_test.sh
\ No newline at end of file
diff --git a/src/test_driver/esp32/CMakeLists.txt b/src/test_driver/esp32/CMakeLists.txt
index bd8756dca7553b..c14242b906908b 100644
--- a/src/test_driver/esp32/CMakeLists.txt
+++ b/src/test_driver/esp32/CMakeLists.txt
@@ -50,3 +50,8 @@ esp32_unit_test(NAME testCore LIBRARY CoreTests)
 esp32_unit_test(NAME testInetLayer LIBRARY InetLayerTests)
 esp32_unit_test(NAME testRetransmit LIBRARY RetransmitTests)
 esp32_unit_test(NAME testSystemLayer LIBRARY SystemLayerTests)
+
+
+# allow other tools to discover what images are available without grepping for '.img'
+string (REPLACE ";" "\n" BUILT_IMAGES "${ESP32_TEST_IMAGES}")
+file(WRITE "${CMAKE_CURRENT_BINARY_DIR}/test_images.txt" "${BUILT_IMAGES}")
diff --git a/src/test_driver/esp32/cmake/esp32_unit_tests.cmake b/src/test_driver/esp32/cmake/esp32_unit_tests.cmake
index 7c0c1f4e65aa3c..26ba66bcbd005b 100644
--- a/src/test_driver/esp32/cmake/esp32_unit_tests.cmake
+++ b/src/test_driver/esp32/cmake/esp32_unit_tests.cmake
@@ -8,6 +8,9 @@
 #    EXTRA_LIBRAIRES - what else to add to link libraries, generally dependencies
 #                      of $LIBRARY   
 #
+# The list ESP32_TEST_IMAGES keeps track of all output images that could
+# be used for testing
+#
 # TODO: several paths are hard-coded here and could use some updates:
 #      - always links to idf::main
 #      - assumes esp-idf/chip/lib is where the built libraries reside
@@ -54,23 +57,36 @@ macro(esp32_unit_test)
       WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}
       COMMENT "Generating binary image from ${UNIT_TEST_NAME}"
     )                                                                                                                                                         
-    add_custom_target(gen_project_binary_${UNIT_TEST_NAME} ALL DEPENDS "${build_dir}/${UNIT_TEST_NAME}.bin_timestamp")
+    add_custom_target(gen_binary_${UNIT_TEST_NAME} ALL DEPENDS "${build_dir}/${UNIT_TEST_NAME}.bin_timestamp")
 
-endmacro()
 
+    ###################### Image executable in QEMU #################
+
+
+    add_custom_command(
+      OUTPUT "${CMAKE_CURRENT_BINARY_DIR}/${UNIT_TEST_NAME}.img_timestamp"
+
+      COMMAND dd if=/dev/zero bs=1024 count=4096 of=${UNIT_TEST_NAME}.img
+      COMMAND dd if=${CMAKE_CURRENT_BINARY_DIR}/bootloader/bootloader.bin bs=1 seek=4096 of=${UNIT_TEST_NAME}.img conv=notrunc
+      COMMAND dd if=${CMAKE_CURRENT_BINARY_DIR}/partition_table/partition-table.bin bs=1 seek=32768 of=${UNIT_TEST_NAME}.img conv=notrunc
+      COMMAND dd if=${CMAKE_CURRENT_BINARY_DIR}/${UNIT_TEST_NAME}.bin bs=1 seek=65536 of=${UNIT_TEST_NAME}.img conv=notrunc
+
+      # $ dd if=build/partition_table/partition-table.bin bs=1 seek=$((0x8000)) of=flash.bin conv=notrunc
+      # $ dd if=build/hello-world.bin bs=1 seek=$((0x10000)) of=flash.bin conv=notrunc
 
-# FIXME: elfbuilder ?
-#    mkdir -p build/chip/
-#    echo "#!/bin/sh" > build/chip/esp32_elf_builder.sh
-#    echo set -e >> build/chip/esp32_elf_builder.sh
-#    echo set -x >> build/chip/esp32_elf_builder.sh
-#    echo $(CXX) $(CXXFLAGS) $(CPPFLAGS) -L$(PROJECT_PATH)/build/chip/lib -Wl,--whole-archive '$$1' -Wl,--no-whole-archive \
-#         -lnlunit-test $(LDFLAGS) -lnlfaultinjection '$$2' -o $(PROJECT_PATH)/build/chip-tests.elf -Wl,-Map=$(APP_MAP) >> build/chip/esp32_elf_builder.sh
+      COMMAND ${CMAKE_COMMAND} -E echo "Generated ${UNIT_TEST_NAME}.img"
+      COMMAND ${CMAKE_COMMAND} -E md5sum "${CMAKE_CURRENT_BINARY_DIR}/${UNIT_TEST_NAME}.img" > "${CMAKE_CURRENT_BINARY_DIR}/${UNIT_TEST_NAME}.img_timestamp"
 
-#    echo $(ESPTOOLPY) elf2image $(ESPTOOL_FLASH_OPTIONS) $(ESPTOOL_ELF2IMAGE_OPTIONS) \
-#         -o $(PROJECT_PATH)/build/chip/chip-tests.bin $(PROJECT_PATH)/build/chip-tests.elf >> build/chip/esp32_elf_builder.sh
-#    ln -sf $(PROJECT_PATH)/build/partitions.bin $(PROJECT_PATH)/build/chip/partitions.bin
-#    mkdir -p build/chip/bootloader
-#    ln -sf $(PROJECT_PATH)/build/bootloader/bootloader.bin $(PROJECT_PATH)/build/chip/bootloader.bin
-#    ln -sf $(PROJECT_PATH)/idf.sh $(PROJECT_PATH)/build/chip/env.sh
+      DEPENDS gen_binary_${UNIT_TEST_NAME}
+      VERBATIM                                                                                                                                                  
+      WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}
+      COMMENT "Generating binary image from ${UNIT_TEST_NAME}"
+    )                                                                                                                                                         
+    add_custom_target(gen_image_${UNIT_TEST_NAME} ALL DEPENDS "${build_dir}/${UNIT_TEST_NAME}.img_timestamp")
 
+    LIST(APPEND ESP32_TEST_IMAGES ${UNIT_TEST_NAME}.img)
+
+
+    # IMAGE CAN BE RUN AS
+    # $QEMU_ESP32 -nographic -no-reboot -machine esp32 -drive file=out/esp32-qemu-tests/testASN1.img,if=mtd,format=raw
+endmacro()
diff --git a/src/test_driver/esp32/qemu_setup.sh b/src/test_driver/esp32/qemu_setup.sh
deleted file mode 100755
index fac58a636271d7..00000000000000
--- a/src/test_driver/esp32/qemu_setup.sh
+++ /dev/null
@@ -1,40 +0,0 @@
-#!/usr/bin/env bash
-#
-#
-#    Copyright (c) 2020 Project CHIP Authors
-#
-#    Licensed under the Apache License, Version 2.0 (the "License");
-#    you may not use this file except in compliance with the License.
-#    You may obtain a copy of the License at
-#
-#        http://www.apache.org/licenses/LICENSE-2.0
-#
-#    Unless required by applicable law or agreed to in writing, software
-#    distributed under the License is distributed on an "AS IS" BASIS,
-#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#    See the License for the specific language governing permissions and
-#    limitations under the License.
-#
-#
-#    Description:
-#      This is a utility script that compiles ESP32 QEMU and sets it up
-#      for testing.
-#
-
-here=$(cd "$(dirname "$0")" && pwd)
-
-set -e
-
-die() {
-    echo "${me:?}: *** ERROR: " "${*}"
-    exit 1
-}
-
-# move to the example folder, I don't work anywhere else
-cd "$here" || die 'ack!, where am I?!?'
-
-# shellcheck source=/dev/null
-source idf.sh
-rm -f ./build/sdkconfig
-SDKCONFIG=./build/sdkconfig SDKCONFIG_DEFAULTS=sdkconfig_qemu.defaults idf make defconfig
-SDKCONFIG=./build/sdkconfig idf make -j8 esp32_elf_builder
diff --git a/src/transport/tests/qemu_transport_tests.sh b/src/transport/tests/qemu_transport_tests.sh
deleted file mode 120000
index 8fbe617c97a267..00000000000000
--- a/src/transport/tests/qemu_transport_tests.sh
+++ /dev/null
@@ -1 +0,0 @@
-../../../scripts/tools/qemu_run_test.sh
\ No newline at end of file

From b831d13b10fa9fbbd1881c0271d2f94d26717e69 Mon Sep 17 00:00:00 2001
From: Andrei Litvin <andy314@gmail.com>
Date: Thu, 16 Dec 2021 15:25:50 -0500
Subject: [PATCH 11/22] Remove obsolete makefiles

---
 src/test_driver/esp32/Makefile          | 30 -------------------------
 src/test_driver/esp32/main/component.mk | 22 ------------------
 2 files changed, 52 deletions(-)
 delete mode 100644 src/test_driver/esp32/Makefile
 delete mode 100644 src/test_driver/esp32/main/component.mk

diff --git a/src/test_driver/esp32/Makefile b/src/test_driver/esp32/Makefile
deleted file mode 100644
index f9f19c729c574f..00000000000000
--- a/src/test_driver/esp32/Makefile
+++ /dev/null
@@ -1,30 +0,0 @@
-#
-# This is a project Makefile. It is assumed the directory this Makefile resides in is a
-# project subdirectory.
-#
-
-PROJECT_NAME := chip-tests
-
-CXXFLAGS += -DCHIP_SUPPORT_FOREIGN_TEST_DRIVERS -DCHIP_TARGET_STYLE_EMBEDDED -Wno-deprecated-declarations
-
-CXXFLAGS += -DLWIP_IPV6_SCOPES=0 -std=gnu++14
-CPPFLAGS += -DLWIP_IPV6_SCOPES=0 -DCHIP_HAVE_CONFIG_H
-CFLAGS += -DLWIP_IPV6_SCOPES=0 -std=gnu11
-
-EXTRA_COMPONENT_DIRS += $(PROJECT_PATH)/third_party/connectedhomeip/config/esp32/components
-
-include $(IDF_PATH)/make/project.mk
-
-esp32_elf_builder: all
-	mkdir -p build/chip/
-	echo "#!/bin/sh" > build/chip/esp32_elf_builder.sh
-	echo set -e >> build/chip/esp32_elf_builder.sh
-	echo set -x >> build/chip/esp32_elf_builder.sh
-	echo $(CXX) $(CXXFLAGS) $(CPPFLAGS) -L$(PROJECT_PATH)/build/chip/lib -Wl,--whole-archive '$$1' -Wl,--no-whole-archive \
-		 -lnlunit-test $(LDFLAGS) -lnlfaultinjection '$$2' -o $(PROJECT_PATH)/build/chip-tests.elf -Wl,-Map=$(APP_MAP) >> build/chip/esp32_elf_builder.sh
-	echo $(ESPTOOLPY) elf2image $(ESPTOOL_FLASH_OPTIONS) $(ESPTOOL_ELF2IMAGE_OPTIONS) \
-		 -o $(PROJECT_PATH)/build/chip/chip-tests.bin $(PROJECT_PATH)/build/chip-tests.elf >> build/chip/esp32_elf_builder.sh
-	ln -sf $(PROJECT_PATH)/build/partitions.bin $(PROJECT_PATH)/build/chip/partitions.bin
-	mkdir -p build/chip/bootloader
-	ln -sf $(PROJECT_PATH)/build/bootloader/bootloader.bin $(PROJECT_PATH)/build/chip/bootloader.bin
-	ln -sf $(PROJECT_PATH)/idf.sh $(PROJECT_PATH)/build/chip/env.sh
diff --git a/src/test_driver/esp32/main/component.mk b/src/test_driver/esp32/main/component.mk
deleted file mode 100644
index e00ee9dea96ffc..00000000000000
--- a/src/test_driver/esp32/main/component.mk
+++ /dev/null
@@ -1,22 +0,0 @@
-#
-#    Copyright (c) 2020 Project CHIP Authors
-#    All rights reserved.
-#
-#    Licensed under the Apache License, Version 2.0 (the "License");
-#    you may not use this file except in compliance with the License.
-#    You may obtain a copy of the License at
-#
-#        http://www.apache.org/licenses/LICENSE-2.0
-#
-#    Unless required by applicable law or agreed to in writing, software
-#    distributed under the License is distributed on an "AS IS" BASIS,
-#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#    See the License for the specific language governing permissions and
-#    limitations under the License.
-#
-#    Description:
-#      Component makefile for the ESP32 demo application.
-#
-# (Uses default behaviour of compiling all source files in directory, adding 'include' to include path.)
-
-COMPONENT_DEPENDS := chip

From a752e5b9f30a702183c85b75f1035d3b133042a8 Mon Sep 17 00:00:00 2001
From: Andrei Litvin <andy314@gmail.com>
Date: Thu, 16 Dec 2021 15:52:09 -0500
Subject: [PATCH 12/22] Switch unit test execution to a python script

---
 .github/workflows/qemu.yaml             | 16 +++-
 src/test_driver/esp32/run_qemu_image.py | 97 +++++++++++++++++++++++++
 2 files changed, 110 insertions(+), 3 deletions(-)
 create mode 100755 src/test_driver/esp32/run_qemu_image.py

diff --git a/.github/workflows/qemu.yaml b/.github/workflows/qemu.yaml
index 239db16b6fa2bd..07751eb1a045b9 100644
--- a/.github/workflows/qemu.yaml
+++ b/.github/workflows/qemu.yaml
@@ -54,9 +54,19 @@ jobs:
                   path: |
                    .environment/gn_out/.ninja_log
                    .environment/pigweed-venv/*.log
-            - name: Build ESP32 QEMU and Run Tests
-              timeout-minutes: 35
-              run: scripts/tests/esp32_qemu_tests.sh /tmp/test_logs
+            - name: Build ESP32 QEMU test images
+              timeout-minutes: 20
+              run: |
+                  scripts/run_in_build_env.sh "         \
+                      ./scripts/build/build_examples.py \
+                      --target esp32-qemu-tests         \
+                      build                             \
+                  "
+            - name: Run all tests
+              timeout-minutes: 30
+              run: |
+                  src/test_driver/esp32/run_qemu_image.py \
+                    --file-image-list ./out/esp32-qemu-tests/test_images.txt
             - name: Uploading Logs
               uses: actions/upload-artifact@v2
               if: ${{ !env.ACT }}
diff --git a/src/test_driver/esp32/run_qemu_image.py b/src/test_driver/esp32/run_qemu_image.py
new file mode 100755
index 00000000000000..4b5d991f9dc63d
--- /dev/null
+++ b/src/test_driver/esp32/run_qemu_image.py
@@ -0,0 +1,97 @@
+#!/usr/bin/env python
+# Copyright (c) 2021 Project CHIP Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import coloredlogs
+import click
+import logging
+import os
+import sys
+import subprocess
+
+# Supported log levels, mapping string values required for argument
+# parsing into logging constants
+__LOG_LEVELS__ = {
+    'debug': logging.DEBUG,
+    'info': logging.INFO,
+    'warn': logging.WARN,
+    'fatal': logging.FATAL,
+}
+
+@click.command()
+@click.option(
+    '--log-level',
+    default='INFO',
+    type=click.Choice(__LOG_LEVELS__.keys(), case_sensitive=False),
+    help='Determines the verbosity of script output.')
+@click.option(
+    '--no-log-timestamps',
+    default=False,
+    is_flag=True,
+    help='Skip timestaps in log output')
+@click.option(
+    '--image',
+    default=[],
+    multiple=True,
+    help='What images to execute (will be executed one after another)'
+)
+@click.option(
+    '--file-image-list',
+    default=None,
+    help='Read the images from the given file (contains images one per line)'
+)
+@click.option(
+    '--qemu',
+    default=os.environ.get('QEMU_ESP32', 'qemu-system-xtensa'),
+    help='QEMU binary to run (generally path to qemu-system-xtensa)'
+)
+def main(log_level, no_log_timestamps, image, file_image_list, qemu):
+    # Ensures somewhat pretty logging of what is going on
+    log_fmt = '%(asctime)s %(levelname)-7s %(message)s'
+    if no_log_timestamps:
+        log_fmt = '%(levelname)-7s %(message)s'
+    coloredlogs.install(level=__LOG_LEVELS__[log_level], fmt=log_fmt)
+
+    image = list(image)
+
+    if file_image_list:
+        logging.info("Reading image list from %s", file_image_list)
+
+        basedir = os.path.dirname(file_image_list)
+
+        with open(file_image_list, 'rt') as f:
+            for name in f.readlines():
+                name = name.strip()
+
+                image_path = name
+                if not os.path.isabs(name):
+                    image_path = os.path.join(basedir, name)
+
+                logging.info("    Found %s => %s", name, image_path)
+                image.append(image_path)
+
+
+    # the list "image" contains all the images that need to run
+    for path in image:
+        logging.info("Executing image %s", path)
+
+        status = subprocess.run([qemu, "-nographic", "-no-reboot", "-machine", "esp32",
+            "-drive", "file=%s,if=mtd,format=raw" % path])
+
+        if status.returncode != 0:
+            raise Exception("Execution of %s failed with code %d" % (path, status.returncode))
+
+
+if __name__ == '__main__':
+    main()
\ No newline at end of file

From e70d8a4f12c5f6aae247d58cb51e0390e37ca096 Mon Sep 17 00:00:00 2001
From: Andrei Litvin <andy314@gmail.com>
Date: Thu, 16 Dec 2021 15:53:59 -0500
Subject: [PATCH 13/22] Some comments and cleanup

---
 src/test_driver/esp32/cmake/esp32_unit_tests.cmake | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/src/test_driver/esp32/cmake/esp32_unit_tests.cmake b/src/test_driver/esp32/cmake/esp32_unit_tests.cmake
index 26ba66bcbd005b..a1f2f9b6058ad1 100644
--- a/src/test_driver/esp32/cmake/esp32_unit_tests.cmake
+++ b/src/test_driver/esp32/cmake/esp32_unit_tests.cmake
@@ -63,6 +63,10 @@ macro(esp32_unit_test)
     ###################### Image executable in QEMU #################
 
 
+    # A runnable image is a 4MB file with:
+    #   bootloader at 0x1000
+    #   partition table at 0x8000
+    #   image at 0x10000
     add_custom_command(
       OUTPUT "${CMAKE_CURRENT_BINARY_DIR}/${UNIT_TEST_NAME}.img_timestamp"
 
@@ -71,9 +75,6 @@ macro(esp32_unit_test)
       COMMAND dd if=${CMAKE_CURRENT_BINARY_DIR}/partition_table/partition-table.bin bs=1 seek=32768 of=${UNIT_TEST_NAME}.img conv=notrunc
       COMMAND dd if=${CMAKE_CURRENT_BINARY_DIR}/${UNIT_TEST_NAME}.bin bs=1 seek=65536 of=${UNIT_TEST_NAME}.img conv=notrunc
 
-      # $ dd if=build/partition_table/partition-table.bin bs=1 seek=$((0x8000)) of=flash.bin conv=notrunc
-      # $ dd if=build/hello-world.bin bs=1 seek=$((0x10000)) of=flash.bin conv=notrunc
-
       COMMAND ${CMAKE_COMMAND} -E echo "Generated ${UNIT_TEST_NAME}.img"
       COMMAND ${CMAKE_COMMAND} -E md5sum "${CMAKE_CURRENT_BINARY_DIR}/${UNIT_TEST_NAME}.img" > "${CMAKE_CURRENT_BINARY_DIR}/${UNIT_TEST_NAME}.img_timestamp"
 

From c7f36ef5f74c55bd0f4dea5766cc8ccf532f7b80 Mon Sep 17 00:00:00 2001
From: Andrei Litvin <andy314@gmail.com>
Date: Thu, 16 Dec 2021 15:54:19 -0500
Subject: [PATCH 14/22] Restyle files

---
 scripts/build/builders/esp32.py         | 7 ++++---
 src/test_driver/esp32/run_qemu_image.py | 9 +++++----
 2 files changed, 9 insertions(+), 7 deletions(-)

diff --git a/scripts/build/builders/esp32.py b/scripts/build/builders/esp32.py
index e05042161ce982..d4ce65ded5dca3 100644
--- a/scripts/build/builders/esp32.py
+++ b/scripts/build/builders/esp32.py
@@ -74,7 +74,7 @@ def AppNamePrefix(self):
     def FlashBundleName(self):
         return self.AppNamePrefix + '.flashbundle.txt'
 
-    def IsCompatible(self, board:Esp32Board):
+    def IsCompatible(self, board: Esp32Board):
         if board == Esp32Board.QEMU:
             return self == Esp32App.TESTS
         elif board == Esp32Board.M5Stack:
@@ -82,7 +82,7 @@ def IsCompatible(self, board:Esp32Board):
         elif board == Esp32Board.C3DevKit:
             return self == Esp32App.ALL_CLUSTERS
         else:
-            return (board == Esp32Board.DevKitC) and (self  != Esp32App.TESTS)
+            return (board == Esp32Board.DevKitC) and (self != Esp32App.TESTS)
 
 
 def DefaultsFileName(board: Esp32Board, app: Esp32App, enable_rpcs: bool):
@@ -119,7 +119,8 @@ def __init__(self,
         self.enable_ipv4 = enable_ipv4
 
         if not app.IsCompatible(board):
-            raise Exception("Incompatible app/board combination: %r and %r", app, board)
+            raise Exception(
+                "Incompatible app/board combination: %r and %r", app, board)
 
     def _IdfEnvExecute(self, cmd, title=None):
         # Run activate.sh after export.sh to ensure using the chip environment.
diff --git a/src/test_driver/esp32/run_qemu_image.py b/src/test_driver/esp32/run_qemu_image.py
index 4b5d991f9dc63d..52be3fcf13fbc3 100755
--- a/src/test_driver/esp32/run_qemu_image.py
+++ b/src/test_driver/esp32/run_qemu_image.py
@@ -29,6 +29,7 @@
     'fatal': logging.FATAL,
 }
 
+
 @click.command()
 @click.option(
     '--log-level',
@@ -81,17 +82,17 @@ def main(log_level, no_log_timestamps, image, file_image_list, qemu):
                 logging.info("    Found %s => %s", name, image_path)
                 image.append(image_path)
 
-
     # the list "image" contains all the images that need to run
     for path in image:
         logging.info("Executing image %s", path)
 
         status = subprocess.run([qemu, "-nographic", "-no-reboot", "-machine", "esp32",
-            "-drive", "file=%s,if=mtd,format=raw" % path])
+                                 "-drive", "file=%s,if=mtd,format=raw" % path])
 
         if status.returncode != 0:
-            raise Exception("Execution of %s failed with code %d" % (path, status.returncode))
+            raise Exception("Execution of %s failed with code %d" %
+                            (path, status.returncode))
 
 
 if __name__ == '__main__':
-    main()
\ No newline at end of file
+    main()

From 1cb85c1bcc01037867ff8a71854ac913a90a9764 Mon Sep 17 00:00:00 2001
From: Andrei Litvin <andy314@gmail.com>
Date: Thu, 16 Dec 2021 16:09:55 -0500
Subject: [PATCH 15/22] Upate build rules and logic - we detect failing and
 passing tests now

---
 .../esp32/cmake/esp32_unit_tests.cmake        |  7 ++++
 src/test_driver/esp32/run_qemu_image.py       | 34 ++++++++++++++++---
 2 files changed, 37 insertions(+), 4 deletions(-)

diff --git a/src/test_driver/esp32/cmake/esp32_unit_tests.cmake b/src/test_driver/esp32/cmake/esp32_unit_tests.cmake
index a1f2f9b6058ad1..62abc5fba42934 100644
--- a/src/test_driver/esp32/cmake/esp32_unit_tests.cmake
+++ b/src/test_driver/esp32/cmake/esp32_unit_tests.cmake
@@ -39,6 +39,13 @@ macro(esp32_unit_test)
         nlunit-test
         nlfaultinjection
     )
+
+    add_dependencies(${UNIT_TEST_NAME} idf::main)
+    add_dependencies(${UNIT_TEST_NAME} idf::chip)
+    # TODO:
+    #   - this does NOT properly handle dependencies  on UNIT_TEST_LIBRARY and such,
+    #     so changes in the tests themselves will not re-gen
+
     idf_build_executable(${UNIT_TEST_NAME})
 
     ######################## flashable image #######################
diff --git a/src/test_driver/esp32/run_qemu_image.py b/src/test_driver/esp32/run_qemu_image.py
index 52be3fcf13fbc3..c92ef144c4ffdf 100755
--- a/src/test_driver/esp32/run_qemu_image.py
+++ b/src/test_driver/esp32/run_qemu_image.py
@@ -87,11 +87,37 @@ def main(log_level, no_log_timestamps, image, file_image_list, qemu):
         logging.info("Executing image %s", path)
 
         status = subprocess.run([qemu, "-nographic", "-no-reboot", "-machine", "esp32",
-                                 "-drive", "file=%s,if=mtd,format=raw" % path])
+                                 "-drive", "file=%s,if=mtd,format=raw" % path], capture_output=True)
 
-        if status.returncode != 0:
-            raise Exception("Execution of %s failed with code %d" %
-                            (path, status.returncode))
+        # Encoding is NOT valid, but want to not try to decode potential
+        # invalid UTF-8 sequences. The strings we care about are ascii anyway
+        output = status.stdout.decode('ascii')
+
+        try:
+            if status.returncode != 0:
+                raise Exception("Execution of %s failed with code %d" %
+                                (path, status.returncode))
+
+            # Parse output of the unit test. Generally expect things like:
+            # Failed Tests:   0 / 5
+            # Failed Asserts: 0 / 77
+            # I (3034) CHIP-tests: CHIP test status: 0
+            for line in output.split('\n'):
+                if line.startswith('Failed Tests:') and not line.startswith('Failed Tests:   0 /'):
+                    raise Exception("Tests seem failed: %s" % line)
+
+                if line.startswith('Failed Asserts:') and not line.startswith('Failed Asserts: 0 /'):
+                    raise Exception("Asserts seem failed: %s" % line)
+
+                if 'CHIP test status: ' in line and 'CHIP test status: 0' not in line:
+                    raise Exception("CHIP test status is NOT 0: %s" % line)
+
+
+            logging.info("Image %s PASSED", path)
+        except:
+            # make sure output is visible in stdout
+            print(output)
+            raise
 
 
 if __name__ == '__main__':

From c61625148f6330298c258b4143a77f8f9e7c7817 Mon Sep 17 00:00:00 2001
From: Andrei Litvin <andy314@gmail.com>
Date: Thu, 16 Dec 2021 16:12:04 -0500
Subject: [PATCH 16/22] Add a verbose test argument

---
 .github/workflows/qemu.yaml             |  1 +
 src/test_driver/esp32/run_qemu_image.py | 11 ++++++++++-
 2 files changed, 11 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/qemu.yaml b/.github/workflows/qemu.yaml
index 07751eb1a045b9..a9069ae033bae5 100644
--- a/.github/workflows/qemu.yaml
+++ b/.github/workflows/qemu.yaml
@@ -59,6 +59,7 @@ jobs:
               run: |
                   scripts/run_in_build_env.sh "         \
                       ./scripts/build/build_examples.py \
+                      --verbose                         \
                       --target esp32-qemu-tests         \
                       build                             \
                   "
diff --git a/src/test_driver/esp32/run_qemu_image.py b/src/test_driver/esp32/run_qemu_image.py
index c92ef144c4ffdf..bf1c2e37480292 100755
--- a/src/test_driver/esp32/run_qemu_image.py
+++ b/src/test_driver/esp32/run_qemu_image.py
@@ -57,7 +57,12 @@
     default=os.environ.get('QEMU_ESP32', 'qemu-system-xtensa'),
     help='QEMU binary to run (generally path to qemu-system-xtensa)'
 )
-def main(log_level, no_log_timestamps, image, file_image_list, qemu):
+@click.option(
+    '--verbose',
+    default=False,
+    is_flag=True,
+    help='More verbose output')
+def main(log_level, no_log_timestamps, image, file_image_list, qemu, verbose):
     # Ensures somewhat pretty logging of what is going on
     log_fmt = '%(asctime)s %(levelname)-7s %(message)s'
     if no_log_timestamps:
@@ -112,6 +117,10 @@ def main(log_level, no_log_timestamps, image, file_image_list, qemu):
                 if 'CHIP test status: ' in line and 'CHIP test status: 0' not in line:
                     raise Exception("CHIP test status is NOT 0: %s" % line)
 
+            if verbose:
+               print("========== TEST OUTPUT BEGIN ============")
+               print(output)
+               print("========== TEST OUTPUT END   ============")
 
             logging.info("Image %s PASSED", path)
         except:

From 557a55aa61fe7cbfdec68639b0b80fd464f7183b Mon Sep 17 00:00:00 2001
From: Andrei Litvin <andy314@gmail.com>
Date: Thu, 16 Dec 2021 16:14:54 -0500
Subject: [PATCH 17/22] Restyle files

---
 src/test_driver/esp32/run_qemu_image.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/test_driver/esp32/run_qemu_image.py b/src/test_driver/esp32/run_qemu_image.py
index bf1c2e37480292..c1bdc186aa6796 100755
--- a/src/test_driver/esp32/run_qemu_image.py
+++ b/src/test_driver/esp32/run_qemu_image.py
@@ -118,9 +118,9 @@ def main(log_level, no_log_timestamps, image, file_image_list, qemu, verbose):
                     raise Exception("CHIP test status is NOT 0: %s" % line)
 
             if verbose:
-               print("========== TEST OUTPUT BEGIN ============")
-               print(output)
-               print("========== TEST OUTPUT END   ============")
+                print("========== TEST OUTPUT BEGIN ============")
+                print(output)
+                print("========== TEST OUTPUT END   ============")
 
             logging.info("Image %s PASSED", path)
         except:

From 928fb44a0aa22da7c20a0170c045d64016e3474e Mon Sep 17 00:00:00 2001
From: Andrei Litvin <andy314@gmail.com>
Date: Thu, 16 Dec 2021 16:22:18 -0500
Subject: [PATCH 18/22] Fix flashbundle logic

---
 scripts/build/builders/esp32.py | 18 +++++++++++++++++-
 1 file changed, 17 insertions(+), 1 deletion(-)

diff --git a/scripts/build/builders/esp32.py b/scripts/build/builders/esp32.py
index d4ce65ded5dca3..e99ab8e291b1b7 100644
--- a/scripts/build/builders/esp32.py
+++ b/scripts/build/builders/esp32.py
@@ -66,12 +66,15 @@ def AppNamePrefix(self):
         elif self == Esp32App.TEMPERATURE_MEASUREMENT:
             return 'chip-temperature-measurement-app'
         elif self == Esp32App.TESTS:
-            return 'FIXME_FIXME_FIXME'
+            return None
         else:
             raise Exception('Unknown app type: %r' % self)
 
     @property
     def FlashBundleName(self):
+        if not self.AppNamePrefix:
+            return None
+
         return self.AppNamePrefix + '.flashbundle.txt'
 
     def IsCompatible(self, board: Esp32Board):
@@ -185,6 +188,16 @@ def _build(self):
         self._IdfEnvExecute(cmd, title='Building ' + self.identifier)
 
     def build_outputs(self):
+        if self.app == Esp32App.TESTS:
+            # Include the runnable image names as artifacts
+            result = dict()
+            with open(os.path.join(self.output_dir, 'test_images.txt'), 'rt') as f:
+                for name in f.readlines():
+                    name = name.strip()
+                    result[name] = os.path.join(self.output_dir, name)
+
+            return result
+
         return {
             self.app.AppNamePrefix + '.elf':
                 os.path.join(self.output_dir, self.app.AppNamePrefix + '.elf'),
@@ -193,6 +206,9 @@ def build_outputs(self):
         }
 
     def flashbundle(self):
+        if not self.app.FlashBundleName:
+            return {}
+
         with open(os.path.join(self.output_dir, self.app.FlashBundleName), 'r') as fp:
             return {
                 l.strip(): os.path.join(self.output_dir, l.strip()) for l in fp.readlines() if l.strip()

From 8d176aa270bc3cd548af684dcc2f9a3f3c5fc4fd Mon Sep 17 00:00:00 2001
From: Andrei Litvin <andy314@gmail.com>
Date: Thu, 16 Dec 2021 16:46:28 -0500
Subject: [PATCH 19/22] Correct placement of verbose argument

---
 .github/workflows/qemu.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/qemu.yaml b/.github/workflows/qemu.yaml
index a9069ae033bae5..c1944954a91702 100644
--- a/.github/workflows/qemu.yaml
+++ b/.github/workflows/qemu.yaml
@@ -59,7 +59,6 @@ jobs:
               run: |
                   scripts/run_in_build_env.sh "         \
                       ./scripts/build/build_examples.py \
-                      --verbose                         \
                       --target esp32-qemu-tests         \
                       build                             \
                   "
@@ -67,6 +66,7 @@ jobs:
               timeout-minutes: 30
               run: |
                   src/test_driver/esp32/run_qemu_image.py \
+                     -verbose                             \
                     --file-image-list ./out/esp32-qemu-tests/test_images.txt
             - name: Uploading Logs
               uses: actions/upload-artifact@v2

From 8e05421ce31021318ae1b605d1a373408467bf18 Mon Sep 17 00:00:00 2001
From: Andrei Litvin <andy314@gmail.com>
Date: Thu, 16 Dec 2021 16:46:55 -0500
Subject: [PATCH 20/22] Correct placement of verbose argument

---
 .github/workflows/qemu.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/qemu.yaml b/.github/workflows/qemu.yaml
index c1944954a91702..b4a340f7d525df 100644
--- a/.github/workflows/qemu.yaml
+++ b/.github/workflows/qemu.yaml
@@ -66,7 +66,7 @@ jobs:
               timeout-minutes: 30
               run: |
                   src/test_driver/esp32/run_qemu_image.py \
-                     -verbose                             \
+                    --verbose                             \
                     --file-image-list ./out/esp32-qemu-tests/test_images.txt
             - name: Uploading Logs
               uses: actions/upload-artifact@v2

From 3c9fc035dec9046ff650c5ba6fde5be13309f8a3 Mon Sep 17 00:00:00 2001
From: Andrei Litvin <andy314@gmail.com>
Date: Thu, 16 Dec 2021 16:55:56 -0500
Subject: [PATCH 21/22] Run test runner with python3 since that is available
 (just python will not work in our build image)

---
 src/test_driver/esp32/run_qemu_image.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/test_driver/esp32/run_qemu_image.py b/src/test_driver/esp32/run_qemu_image.py
index c1bdc186aa6796..d871083a0c1c77 100755
--- a/src/test_driver/esp32/run_qemu_image.py
+++ b/src/test_driver/esp32/run_qemu_image.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 # Copyright (c) 2021 Project CHIP Authors
 #
 # Licensed under the Apache License, Version 2.0 (the "License");

From 5ec48c6575fd0c4d1a085b2e91f3349d63a82b24 Mon Sep 17 00:00:00 2001
From: Andrei Litvin <andy314@gmail.com>
Date: Thu, 16 Dec 2021 17:14:53 -0500
Subject: [PATCH 22/22] Update build example py unit tests

---
 .../build/testdata/all_targets_except_host.txt |  1 +
 .../build/testdata/build_all_except_host.txt   | 18 ++++++++++++++++++
 .../testdata/glob_star_targets_except_host.txt |  1 +
 3 files changed, 20 insertions(+)

diff --git a/scripts/build/testdata/all_targets_except_host.txt b/scripts/build/testdata/all_targets_except_host.txt
index 8c803ff3f2a1da..eb986fae869f64 100644
--- a/scripts/build/testdata/all_targets_except_host.txt
+++ b/scripts/build/testdata/all_targets_except_host.txt
@@ -70,6 +70,7 @@ esp32-m5stack-all-clusters
 esp32-m5stack-all-clusters-ipv6only
 esp32-m5stack-all-clusters-rpc
 esp32-m5stack-all-clusters-rpc-ipv6only
+esp32-qemu-tests
 infineon-p6-all-clusters
 infineon-p6-light
 infineon-p6-lock
diff --git a/scripts/build/testdata/build_all_except_host.txt b/scripts/build/testdata/build_all_except_host.txt
index cbe3c7d9d98b0c..e1df7360aa8336 100644
--- a/scripts/build/testdata/build_all_except_host.txt
+++ b/scripts/build/testdata/build_all_except_host.txt
@@ -410,6 +410,17 @@ bash -c 'source $IDF_PATH/export.sh; source scripts/activate.sh;
 export SDKCONFIG_DEFAULTS={out}/esp32-m5stack-all-clusters-rpc-ipv6only/sdkconfig.defaults
 idf.py -C examples/all-clusters-app/esp32 -B {out}/esp32-m5stack-all-clusters-rpc-ipv6only reconfigure'
 
+# Generating esp32-qemu-tests
+mkdir -p {out}/esp32-qemu-tests
+
+cp src/test_driver/esp32/sdkconfig_qemu.defaults {out}/esp32-qemu-tests/sdkconfig.defaults
+
+rm -f src/test_driver/esp32/sdkconfig
+
+bash -c 'source $IDF_PATH/export.sh; source scripts/activate.sh; 
+export SDKCONFIG_DEFAULTS={out}/esp32-qemu-tests/sdkconfig.defaults
+idf.py -C src/test_driver/esp32 -B {out}/esp32-qemu-tests reconfigure'
+
 # Generating infineon-p6-all-clusters
 gn gen --check --fail-on-unused-args --export-compile-commands --root={root}/examples/all-clusters-app/p6 '--args=p6_board="CY8CKIT-062S2-43012"' {out}/infineon-p6-all-clusters
 
@@ -1002,6 +1013,13 @@ bash -c 'source $IDF_PATH/export.sh; source scripts/activate.sh;
 export SDKCONFIG_DEFAULTS={out}/esp32-m5stack-all-clusters-rpc-ipv6only/sdkconfig.defaults
 idf.py -C examples/all-clusters-app/esp32 -B {out}/esp32-m5stack-all-clusters-rpc-ipv6only build'
 
+rm -f src/test_driver/esp32/sdkconfig
+
+# Building esp32-qemu-tests
+bash -c 'source $IDF_PATH/export.sh; source scripts/activate.sh; 
+export SDKCONFIG_DEFAULTS={out}/esp32-qemu-tests/sdkconfig.defaults
+idf.py -C src/test_driver/esp32 -B {out}/esp32-qemu-tests build'
+
 # Building infineon-p6-all-clusters
 ninja -C {out}/infineon-p6-all-clusters
 
diff --git a/scripts/build/testdata/glob_star_targets_except_host.txt b/scripts/build/testdata/glob_star_targets_except_host.txt
index 9e7403d0f346c7..b478d6c827af0e 100644
--- a/scripts/build/testdata/glob_star_targets_except_host.txt
+++ b/scripts/build/testdata/glob_star_targets_except_host.txt
@@ -28,6 +28,7 @@ esp32-m5stack-all-clusters
 esp32-m5stack-all-clusters-ipv6only
 esp32-m5stack-all-clusters-rpc
 esp32-m5stack-all-clusters-rpc-ipv6only
+esp32-qemu-tests
 infineon-p6-all-clusters
 infineon-p6-light
 infineon-p6-lock