From 2519988ad3dfdd22a90552d56244cbd9d2ac1d5d Mon Sep 17 00:00:00 2001 From: Andrei Litvin Date: Fri, 17 Dec 2021 09:16:24 -0500 Subject: [PATCH] Use CMake and separate runner for QEMU tests for esp32 (#13095) * Start adding a tests target to qemu * Fix extra compatibility test * Prepare qemu builds for the test driver using cmake * Ensure sdkconfig can be ignored for esp32 test driver: this file is re-written by the cmake build * In theory we compile and link at least one test * Tests now build for a few sub-items * Tests now build for a few sub-items * move the cmake logic for esp tests into a separate cmakefile * bin is also built by the cmake system now * Start being able to run test - for now remove the qemu sh scripts * Remove obsolete makefiles * Switch unit test execution to a python script * Some comments and cleanup * Restyle files * Upate build rules and logic - we detect failing and passing tests now * Add a verbose test argument * Restyle files * Fix flashbundle logic * Correct placement of verbose argument * Correct placement of verbose argument * Run test runner with python3 since that is available (just python will not work in our build image) * Update build example py unit tests --- .github/workflows/qemu.yaml | 20 ++- scripts/build/build/targets.py | 2 + scripts/build/builders/esp32.py | 57 ++++++-- .../testdata/all_targets_except_host.txt | 1 + .../build/testdata/build_all_except_host.txt | 18 +++ .../glob_star_targets_except_host.txt | 1 + scripts/tests/esp32_qemu_tests.sh | 91 ------------ scripts/tools/esp32_qemu_run.sh | 74 ---------- scripts/tools/qemu_run_test.sh | 59 -------- src/crypto/tests/qemu_crypto_tests.sh | 1 - src/inet/tests/qemu_inet_tests.sh | 1 - src/system/tests/qemu_system_tests.sh | 1 - src/test_driver/esp32/.gitignore | 2 + src/test_driver/esp32/CMakeLists.txt | 57 ++++++++ src/test_driver/esp32/Makefile | 30 ---- .../esp32/cmake/esp32_unit_tests.cmake | 100 +++++++++++++ src/test_driver/esp32/dummy.c | 0 src/test_driver/esp32/main/CMakeLists.txt | 42 ++++++ src/test_driver/esp32/main/component.mk | 22 --- src/test_driver/esp32/qemu_setup.sh | 40 ------ src/test_driver/esp32/run_qemu_image.py | 133 ++++++++++++++++++ src/test_driver/esp32/sdkconfig_qemu.defaults | 4 +- src/transport/tests/qemu_transport_tests.sh | 1 - 23 files changed, 420 insertions(+), 337 deletions(-) delete mode 100755 scripts/tests/esp32_qemu_tests.sh delete mode 100755 scripts/tools/esp32_qemu_run.sh delete mode 100755 scripts/tools/qemu_run_test.sh delete mode 120000 src/crypto/tests/qemu_crypto_tests.sh delete mode 120000 src/inet/tests/qemu_inet_tests.sh delete mode 120000 src/system/tests/qemu_system_tests.sh create mode 100644 src/test_driver/esp32/CMakeLists.txt delete mode 100644 src/test_driver/esp32/Makefile create mode 100644 src/test_driver/esp32/cmake/esp32_unit_tests.cmake create mode 100644 src/test_driver/esp32/dummy.c create mode 100644 src/test_driver/esp32/main/CMakeLists.txt delete mode 100644 src/test_driver/esp32/main/component.mk delete mode 100755 src/test_driver/esp32/qemu_setup.sh create mode 100755 src/test_driver/esp32/run_qemu_image.py delete mode 120000 src/transport/tests/qemu_transport_tests.sh diff --git a/.github/workflows/qemu.yaml b/.github/workflows/qemu.yaml index 6f68a6409e7ea7..b4a340f7d525df 100644 --- a/.github/workflows/qemu.yaml +++ b/.github/workflows/qemu.yaml @@ -54,12 +54,20 @@ jobs: path: | .environment/gn_out/.ninja_log .environment/pigweed-venv/*.log - - name: Build example All Clusters App - timeout-minutes: 25 - run: scripts/examples/esp_example.sh all-clusters-app - - name: Build ESP32 QEMU and Run Tests - timeout-minutes: 35 - run: scripts/tests/esp32_qemu_tests.sh /tmp/test_logs + - name: Build ESP32 QEMU test images + timeout-minutes: 20 + run: | + scripts/run_in_build_env.sh " \ + ./scripts/build/build_examples.py \ + --target esp32-qemu-tests \ + build \ + " + - name: Run all tests + timeout-minutes: 30 + run: | + src/test_driver/esp32/run_qemu_image.py \ + --verbose \ + --file-image-list ./out/esp32-qemu-tests/test_images.txt - name: Uploading Logs uses: actions/upload-artifact@v2 if: ${{ !env.ACT }} diff --git a/scripts/build/build/targets.py b/scripts/build/build/targets.py index 3dbaa16ea89e83..043fad9c5b1691 100644 --- a/scripts/build/build/targets.py +++ b/scripts/build/build/targets.py @@ -220,6 +220,8 @@ def Esp32Targets(): yield devkitc.Extend('bridge', app=Esp32App.BRIDGE) yield devkitc.Extend('temperature-measurement', app=Esp32App.TEMPERATURE_MEASUREMENT) + yield esp32_target.Extend('qemu-tests', board=Esp32Board.QEMU, app=Esp32App.TESTS) + def Efr32Targets(): efr_target = Target('efr32', Efr32Builder) diff --git a/scripts/build/builders/esp32.py b/scripts/build/builders/esp32.py index f3f709e4e76e83..e99ab8e291b1b7 100644 --- a/scripts/build/builders/esp32.py +++ b/scripts/build/builders/esp32.py @@ -25,6 +25,7 @@ class Esp32Board(Enum): DevKitC = auto() M5Stack = auto() C3DevKit = auto() + QEMU = auto() class Esp32App(Enum): @@ -33,19 +34,22 @@ class Esp32App(Enum): SHELL = auto() BRIDGE = auto() TEMPERATURE_MEASUREMENT = auto() + TESTS = auto() @property - def ExampleName(self): + def ExamplePath(self): if self == Esp32App.ALL_CLUSTERS: - return 'all-clusters-app' + return 'examples/all-clusters-app' elif self == Esp32App.LOCK: - return 'lock-app' + return 'examples/lock-app' elif self == Esp32App.SHELL: - return 'shell' + return 'examples/shell' elif self == Esp32App.BRIDGE: - return 'bridge-app' + return 'examples/bridge-app' elif self == Esp32App.TEMPERATURE_MEASUREMENT: - return 'temperature-measurement-app' + return 'examples/temperature-measurement-app' + elif self == Esp32App.TESTS: + return 'src/test_driver' else: raise Exception('Unknown app type: %r' % self) @@ -61,15 +65,33 @@ def AppNamePrefix(self): return 'chip-bridge-app' elif self == Esp32App.TEMPERATURE_MEASUREMENT: return 'chip-temperature-measurement-app' + elif self == Esp32App.TESTS: + return None else: raise Exception('Unknown app type: %r' % self) + @property def FlashBundleName(self): + if not self.AppNamePrefix: + return None + return self.AppNamePrefix + '.flashbundle.txt' + def IsCompatible(self, board: Esp32Board): + if board == Esp32Board.QEMU: + return self == Esp32App.TESTS + elif board == Esp32Board.M5Stack: + return self == Esp32App.ALL_CLUSTERS + elif board == Esp32Board.C3DevKit: + return self == Esp32App.ALL_CLUSTERS + else: + return (board == Esp32Board.DevKitC) and (self != Esp32App.TESTS) + def DefaultsFileName(board: Esp32Board, app: Esp32App, enable_rpcs: bool): - if app != Esp32App.ALL_CLUSTERS: + if app == Esp32App.TESTS: + return 'sdkconfig_qemu.defaults' + elif app != Esp32App.ALL_CLUSTERS: return 'sdkconfig.defaults' rpc = "_rpc" if enable_rpcs else "" @@ -99,6 +121,10 @@ def __init__(self, self.enable_rpcs = enable_rpcs self.enable_ipv4 = enable_ipv4 + if not app.IsCompatible(board): + raise Exception( + "Incompatible app/board combination: %r and %r", app, board) + def _IdfEnvExecute(self, cmd, title=None): # Run activate.sh after export.sh to ensure using the chip environment. self._Execute( @@ -107,7 +133,7 @@ def _IdfEnvExecute(self, cmd, title=None): @property def ExamplePath(self): - return os.path.join('examples', self.app.ExampleName, 'esp32') + return os.path.join(self.app.ExamplePath, 'esp32') def generate(self): if os.path.exists(os.path.join(self.output_dir, 'build.ninja')): @@ -162,6 +188,16 @@ def _build(self): self._IdfEnvExecute(cmd, title='Building ' + self.identifier) def build_outputs(self): + if self.app == Esp32App.TESTS: + # Include the runnable image names as artifacts + result = dict() + with open(os.path.join(self.output_dir, 'test_images.txt'), 'rt') as f: + for name in f.readlines(): + name = name.strip() + result[name] = os.path.join(self.output_dir, name) + + return result + return { self.app.AppNamePrefix + '.elf': os.path.join(self.output_dir, self.app.AppNamePrefix + '.elf'), @@ -170,7 +206,10 @@ def build_outputs(self): } def flashbundle(self): - with open(os.path.join(self.output_dir, self.app.FlashBundleName()), 'r') as fp: + if not self.app.FlashBundleName: + return {} + + with open(os.path.join(self.output_dir, self.app.FlashBundleName), 'r') as fp: return { l.strip(): os.path.join(self.output_dir, l.strip()) for l in fp.readlines() if l.strip() } diff --git a/scripts/build/testdata/all_targets_except_host.txt b/scripts/build/testdata/all_targets_except_host.txt index 8c803ff3f2a1da..eb986fae869f64 100644 --- a/scripts/build/testdata/all_targets_except_host.txt +++ b/scripts/build/testdata/all_targets_except_host.txt @@ -70,6 +70,7 @@ esp32-m5stack-all-clusters esp32-m5stack-all-clusters-ipv6only esp32-m5stack-all-clusters-rpc esp32-m5stack-all-clusters-rpc-ipv6only +esp32-qemu-tests infineon-p6-all-clusters infineon-p6-light infineon-p6-lock diff --git a/scripts/build/testdata/build_all_except_host.txt b/scripts/build/testdata/build_all_except_host.txt index cbe3c7d9d98b0c..e1df7360aa8336 100644 --- a/scripts/build/testdata/build_all_except_host.txt +++ b/scripts/build/testdata/build_all_except_host.txt @@ -410,6 +410,17 @@ bash -c 'source $IDF_PATH/export.sh; source scripts/activate.sh; export SDKCONFIG_DEFAULTS={out}/esp32-m5stack-all-clusters-rpc-ipv6only/sdkconfig.defaults idf.py -C examples/all-clusters-app/esp32 -B {out}/esp32-m5stack-all-clusters-rpc-ipv6only reconfigure' +# Generating esp32-qemu-tests +mkdir -p {out}/esp32-qemu-tests + +cp src/test_driver/esp32/sdkconfig_qemu.defaults {out}/esp32-qemu-tests/sdkconfig.defaults + +rm -f src/test_driver/esp32/sdkconfig + +bash -c 'source $IDF_PATH/export.sh; source scripts/activate.sh; +export SDKCONFIG_DEFAULTS={out}/esp32-qemu-tests/sdkconfig.defaults +idf.py -C src/test_driver/esp32 -B {out}/esp32-qemu-tests reconfigure' + # Generating infineon-p6-all-clusters gn gen --check --fail-on-unused-args --export-compile-commands --root={root}/examples/all-clusters-app/p6 '--args=p6_board="CY8CKIT-062S2-43012"' {out}/infineon-p6-all-clusters @@ -1002,6 +1013,13 @@ bash -c 'source $IDF_PATH/export.sh; source scripts/activate.sh; export SDKCONFIG_DEFAULTS={out}/esp32-m5stack-all-clusters-rpc-ipv6only/sdkconfig.defaults idf.py -C examples/all-clusters-app/esp32 -B {out}/esp32-m5stack-all-clusters-rpc-ipv6only build' +rm -f src/test_driver/esp32/sdkconfig + +# Building esp32-qemu-tests +bash -c 'source $IDF_PATH/export.sh; source scripts/activate.sh; +export SDKCONFIG_DEFAULTS={out}/esp32-qemu-tests/sdkconfig.defaults +idf.py -C src/test_driver/esp32 -B {out}/esp32-qemu-tests build' + # Building infineon-p6-all-clusters ninja -C {out}/infineon-p6-all-clusters diff --git a/scripts/build/testdata/glob_star_targets_except_host.txt b/scripts/build/testdata/glob_star_targets_except_host.txt index 9e7403d0f346c7..b478d6c827af0e 100644 --- a/scripts/build/testdata/glob_star_targets_except_host.txt +++ b/scripts/build/testdata/glob_star_targets_except_host.txt @@ -28,6 +28,7 @@ esp32-m5stack-all-clusters esp32-m5stack-all-clusters-ipv6only esp32-m5stack-all-clusters-rpc esp32-m5stack-all-clusters-rpc-ipv6only +esp32-qemu-tests infineon-p6-all-clusters infineon-p6-light infineon-p6-lock diff --git a/scripts/tests/esp32_qemu_tests.sh b/scripts/tests/esp32_qemu_tests.sh deleted file mode 100755 index 942c9403962f61..00000000000000 --- a/scripts/tests/esp32_qemu_tests.sh +++ /dev/null @@ -1,91 +0,0 @@ -#!/usr/bin/env bash -# -# -# Copyright (c) 2020 Project CHIP Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# -# Description: -# This is scripts builds ESP32 QEMU, and runs CHIP unit tests using it. -# - -set -e -set -o pipefail -set -x - -here=$(cd "$(dirname "$0")" && pwd) -chip_dir="$here"/../.. - -if [[ -n "$1" ]]; then - log_dir=$1 - shift -fi - -# shellcheck source=/dev/null -source "$chip_dir"/src/test_driver/esp32/idf.sh -"$chip_dir"/src/test_driver/esp32/qemu_setup.sh - -if [ $? -ne 0 ]; then - echo "Setup failure" - exit 1 -fi - -really_run_suite() { - idf scripts/tools/qemu_run_test.sh src/test_driver/esp32/build/chip "$@" -} - -run_suite() { - if [[ -d "${log_dir}" ]]; then - suite=${1%.a} - suite=${suite#lib} - really_run_suite "$@" 2>&1 | tee "$log_dir/$suite.log" - else - really_run_suite "$@" - fi -} - -# Currently only crypto, inet, and system tests are configured to run on QEMU. -# The specific qualifiers will be removed, once all CHIP unit tests are -# updated to run on QEMU. -SUITES=( -) - -# TODO: libAppTests depends on MessagingTestHelpers, which depends on -# NetworkTestHelpers. That sort of depends on InetTestHelpers or -# equivalent (to provide gSystemLayer, gInet, InitNetwork(), -# ShutdownNetwork()) but there's only a POSIX implementation of that -# last, which does not compile on ESP32. Need to figure out how to -# make that work. See comments below for the transport layer tests, -# which have the same issue. -# run_suite libAppTests.a -lMessagingTestHelpers -lNetworkTestHelpers - -run_suite libASN1Tests.a -run_suite libBleLayerTests.a -run_suite libCoreTests.a -run_suite libInetLayerTests.a -run_suite libRetransmitTests.a -run_suite libSystemLayerTests.a -run_suite libChipCryptoTests.a "-lChipCertTestVectors" - -# TODO: Transport layer tests do not link: -# - getpid undefined -# - ArgParser for IPAddresses are not linked in -# - std::__throw_bad_alloc() linker errors -# run_suite libRawTransportTests.a "-lNetworkTestHelpers -lInetTestHelpers" - -# TODO: Transport layer tests do not link: -# - getpid undefined -# - ArgParser for IPAddresses are not linked in -# - std::__throw_bad_alloc() linker errors -# run_suite libTransportLayerTests.a "-lNetworkTestHelpers -lInetTestHelpers" diff --git a/scripts/tools/esp32_qemu_run.sh b/scripts/tools/esp32_qemu_run.sh deleted file mode 100755 index fb4e1e092a99d0..00000000000000 --- a/scripts/tools/esp32_qemu_run.sh +++ /dev/null @@ -1,74 +0,0 @@ -#!/usr/bin/env bash -# -# -# Copyright (c) 2020 Project CHIP Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -# -# Description: -# This is a utility script that runs ESP32 QEMU using the given -# application image. -# - -usage() { - exitcode=0 - if [[ -n "$1" ]]; then - exitcode=1 - echo "*** Error: $*" - fi - echo "Usage: $0 " - exit "$exitcode" -} - -me=$(basename "$0") -die() { - echo "$me: *** ERROR: " "${*}" - exit 1 -} - -realpath() { - path=$1 # input - - [[ -z $path ]] && return 0 - - # trim trailing slashes - while [[ ${#path} -gt 1 && $path = */ ]]; do - path=${path%/} - done - - # if we're at root we're done - if [[ $path = / ]]; then - echo "$path" - return 0 - fi - - [[ $path != /* ]] && path=$PWD/$path - - if [[ -d $path ]]; then - (cd "$path" && pwd) - else - echo "$(realpath "${path%/*}")/${path##*/}" - fi -} - -[[ $# -eq 1 ]] || usage "Incorrect number of arguments" - -[[ -n $QEMU_ESP32 ]] || die "Environment variable QEMU_ESP32 is undefined." - -flash_image=$(realpath "$1") - -[[ -r $flash_image ]] || usage "Could not read file $flash_image" - -"$QEMU_ESP32" -nographic -machine esp32 -drive file="$flash_image",if=mtd,format=raw -no-reboot diff --git a/scripts/tools/qemu_run_test.sh b/scripts/tools/qemu_run_test.sh deleted file mode 100755 index c2196bf7c657b7..00000000000000 --- a/scripts/tools/qemu_run_test.sh +++ /dev/null @@ -1,59 +0,0 @@ -#!/usr/bin/env bash -# -# -# Copyright (c) 2020 Project CHIP Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# -# Description: -# This is a utility script that runs CHIP tests using ESP32 QEMU. -# - -set -e -set -x - -die() { - echo "${me:?}: *** ERROR: " "${*}" - exit 1 -} - -SRC_DIR="$(dirname "$0")/../.." -BUILD_DIR="$1" -shift -QEMU_TEST_TARGET="$1" -shift -EXTRA_COMPILE_ARGUMENTS="$*" # generally -lFooHelperLibrary - -# shellcheck source=/dev/null -source "$BUILD_DIR"/env.sh -bash "$BUILD_DIR"/esp32_elf_builder.sh "$BUILD_DIR/lib/$QEMU_TEST_TARGET" "$EXTRA_COMPILE_ARGUMENTS" - -flash_image_file=$(mktemp) -log_file=$(mktemp) -trap '{ rm -f $flash_image_file $log_file; }' EXIT - -"$SRC_DIR"/scripts/tools/build_esp32_flash_image.sh "$BUILD_DIR"/chip-tests.bin "$flash_image_file" -"$SRC_DIR"/scripts/tools/esp32_qemu_run.sh "$flash_image_file" | tee "$log_file" - -# If the logs contain failure message -if grep -F "] : FAILED" "$log_file"; then - die 'Some tests failed.' -fi - -# If the logs do not contain final success status -if grep -F "CHIP-tests: CHIP test status: 0" "$log_file"; then - echo "$me: All tests passed" -else - die 'Tests did not run to completion.' -fi diff --git a/src/crypto/tests/qemu_crypto_tests.sh b/src/crypto/tests/qemu_crypto_tests.sh deleted file mode 120000 index 8fbe617c97a267..00000000000000 --- a/src/crypto/tests/qemu_crypto_tests.sh +++ /dev/null @@ -1 +0,0 @@ -../../../scripts/tools/qemu_run_test.sh \ No newline at end of file diff --git a/src/inet/tests/qemu_inet_tests.sh b/src/inet/tests/qemu_inet_tests.sh deleted file mode 120000 index 8fbe617c97a267..00000000000000 --- a/src/inet/tests/qemu_inet_tests.sh +++ /dev/null @@ -1 +0,0 @@ -../../../scripts/tools/qemu_run_test.sh \ No newline at end of file diff --git a/src/system/tests/qemu_system_tests.sh b/src/system/tests/qemu_system_tests.sh deleted file mode 120000 index 8fbe617c97a267..00000000000000 --- a/src/system/tests/qemu_system_tests.sh +++ /dev/null @@ -1 +0,0 @@ -../../../scripts/tools/qemu_run_test.sh \ No newline at end of file diff --git a/src/test_driver/esp32/.gitignore b/src/test_driver/esp32/.gitignore index 84c048a73cc2e5..a90ab6f1e57d5a 100644 --- a/src/test_driver/esp32/.gitignore +++ b/src/test_driver/esp32/.gitignore @@ -1 +1,3 @@ /build/ +/sdkconfig +/sdkconfig.old diff --git a/src/test_driver/esp32/CMakeLists.txt b/src/test_driver/esp32/CMakeLists.txt new file mode 100644 index 00000000000000..c14242b906908b --- /dev/null +++ b/src/test_driver/esp32/CMakeLists.txt @@ -0,0 +1,57 @@ +# +# Copyright (c) 2021 Project CHIP Authors +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# The following lines of boilerplate have to be in your project's +# CMakeLists in this exact order for cmake to work correctly +cmake_minimum_required(VERSION 3.5) +include($ENV{IDF_PATH}/tools/cmake/project.cmake) + +include(${CMAKE_CURRENT_SOURCE_DIR}/cmake/esp32_unit_tests.cmake) + +set(EXTRA_COMPONENT_DIRS + "${CMAKE_CURRENT_LIST_DIR}/third_party/connectedhomeip/config/esp32/components" +) + +project(test-driver) + +# C++17 is required for RPC build. +idf_build_set_property(CXX_COMPILE_OPTIONS "-std=gnu++17;-Os;-DLWIP_IPV6_SCOPES=0;-DCHIP_HAVE_CONFIG_H" APPEND) +idf_build_set_property(C_COMPILE_OPTIONS "-Os;-DLWIP_IPV6_SCOPES=0" APPEND) + +# TODO: libAppTests depends on MessagingTestHelpers, which depends on +# NetworkTestHelpers. That sort of depends on InetTestHelpers or +# equivalent (to provide gSystemLayer, gInet, InitNetwork(), +# ShutdownNetwork()) but there's only a POSIX implementation of that +# last, which does not compile on ESP32. Need to figure out how to +# make that work. See comments below for the transport layer tests, +# which have the same issue. +# +# libAppTests.a -lMessagingTestHelpers -lNetworkTestHelpers +# +# TODO: ble tests do not compile using CMake (library is not auto-built) +# libBleLayerTests.a + +esp32_unit_test(NAME testASN1 LIBRARY ASN1Tests) +esp32_unit_test(NAME testChipCrypto LIBRARY ChipCryptoTests EXTRA_LIBRARIES -lChipCertTestVectors) +esp32_unit_test(NAME testCore LIBRARY CoreTests) +esp32_unit_test(NAME testInetLayer LIBRARY InetLayerTests) +esp32_unit_test(NAME testRetransmit LIBRARY RetransmitTests) +esp32_unit_test(NAME testSystemLayer LIBRARY SystemLayerTests) + + +# allow other tools to discover what images are available without grepping for '.img' +string (REPLACE ";" "\n" BUILT_IMAGES "${ESP32_TEST_IMAGES}") +file(WRITE "${CMAKE_CURRENT_BINARY_DIR}/test_images.txt" "${BUILT_IMAGES}") diff --git a/src/test_driver/esp32/Makefile b/src/test_driver/esp32/Makefile deleted file mode 100644 index f9f19c729c574f..00000000000000 --- a/src/test_driver/esp32/Makefile +++ /dev/null @@ -1,30 +0,0 @@ -# -# This is a project Makefile. It is assumed the directory this Makefile resides in is a -# project subdirectory. -# - -PROJECT_NAME := chip-tests - -CXXFLAGS += -DCHIP_SUPPORT_FOREIGN_TEST_DRIVERS -DCHIP_TARGET_STYLE_EMBEDDED -Wno-deprecated-declarations - -CXXFLAGS += -DLWIP_IPV6_SCOPES=0 -std=gnu++14 -CPPFLAGS += -DLWIP_IPV6_SCOPES=0 -DCHIP_HAVE_CONFIG_H -CFLAGS += -DLWIP_IPV6_SCOPES=0 -std=gnu11 - -EXTRA_COMPONENT_DIRS += $(PROJECT_PATH)/third_party/connectedhomeip/config/esp32/components - -include $(IDF_PATH)/make/project.mk - -esp32_elf_builder: all - mkdir -p build/chip/ - echo "#!/bin/sh" > build/chip/esp32_elf_builder.sh - echo set -e >> build/chip/esp32_elf_builder.sh - echo set -x >> build/chip/esp32_elf_builder.sh - echo $(CXX) $(CXXFLAGS) $(CPPFLAGS) -L$(PROJECT_PATH)/build/chip/lib -Wl,--whole-archive '$$1' -Wl,--no-whole-archive \ - -lnlunit-test $(LDFLAGS) -lnlfaultinjection '$$2' -o $(PROJECT_PATH)/build/chip-tests.elf -Wl,-Map=$(APP_MAP) >> build/chip/esp32_elf_builder.sh - echo $(ESPTOOLPY) elf2image $(ESPTOOL_FLASH_OPTIONS) $(ESPTOOL_ELF2IMAGE_OPTIONS) \ - -o $(PROJECT_PATH)/build/chip/chip-tests.bin $(PROJECT_PATH)/build/chip-tests.elf >> build/chip/esp32_elf_builder.sh - ln -sf $(PROJECT_PATH)/build/partitions.bin $(PROJECT_PATH)/build/chip/partitions.bin - mkdir -p build/chip/bootloader - ln -sf $(PROJECT_PATH)/build/bootloader/bootloader.bin $(PROJECT_PATH)/build/chip/bootloader.bin - ln -sf $(PROJECT_PATH)/idf.sh $(PROJECT_PATH)/build/chip/env.sh diff --git a/src/test_driver/esp32/cmake/esp32_unit_tests.cmake b/src/test_driver/esp32/cmake/esp32_unit_tests.cmake new file mode 100644 index 00000000000000..62abc5fba42934 --- /dev/null +++ b/src/test_driver/esp32/cmake/esp32_unit_tests.cmake @@ -0,0 +1,100 @@ + +# Defines a stand-alone elf unit test that can be executed in QEMU +# +# Parameters: +# NAME - the elf file name +# LIBRARY - the library that contains the registered unit tests. +# for libASN1Tests.a use ASN1Tests as name. +# EXTRA_LIBRAIRES - what else to add to link libraries, generally dependencies +# of $LIBRARY +# +# The list ESP32_TEST_IMAGES keeps track of all output images that could +# be used for testing +# +# TODO: several paths are hard-coded here and could use some updates: +# - always links to idf::main +# - assumes esp-idf/chip/lib is where the built libraries reside +# - assumes a "dummy.c" source exists to be able to "add_executable" +macro(esp32_unit_test) + cmake_parse_arguments( + UNIT_TEST + "" # options + "NAME;LIBRARY" # one value arguments + "EXTRA_LIBRARIES" # multi value arguments + ${ARGN} + ) + + ######################## Elf binary ####################### + + add_executable(${UNIT_TEST_NAME} dummy.c) + + target_link_directories(${UNIT_TEST_NAME} PUBLIC + ${CMAKE_CURRENT_BINARY_DIR}/esp-idf/chip/lib + ) + + target_link_libraries(${UNIT_TEST_NAME} PUBLIC + idf::main + -Wl,--whole-archive ${UNIT_TEST_LIBRARY} -Wl,--no-whole-archive + ${UNIT_TEST_EXTRA_LIBRARIES} + nlunit-test + nlfaultinjection + ) + + add_dependencies(${UNIT_TEST_NAME} idf::main) + add_dependencies(${UNIT_TEST_NAME} idf::chip) + # TODO: + # - this does NOT properly handle dependencies on UNIT_TEST_LIBRARY and such, + # so changes in the tests themselves will not re-gen + + idf_build_executable(${UNIT_TEST_NAME}) + + ######################## flashable image ####################### + + # This is very hacky: ESP build system will generate this, but only for the FIRST + # executable that is used + + add_custom_command( + OUTPUT "${CMAKE_CURRENT_BINARY_DIR}/${UNIT_TEST_NAME}.bin_timestamp" + COMMAND ${ESPTOOLPY} elf2image ${ESPTOOLPY_FLASH_OPTIONS} ${esptool_elf2image_args} + -o "${CMAKE_CURRENT_BINARY_DIR}/${UNIT_TEST_NAME}.bin" "${UNIT_TEST_NAME}" + COMMAND ${CMAKE_COMMAND} -E echo "Generated ${UNIT_TEST_NAME}.bin" + COMMAND ${CMAKE_COMMAND} -E md5sum "${CMAKE_CURRENT_BINARY_DIR}/${UNIT_TEST_NAME}" > "${CMAKE_CURRENT_BINARY_DIR}/${UNIT_TEST_NAME}.bin_timestamp" + DEPENDS ${UNIT_TEST_NAME} + VERBATIM + WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} + COMMENT "Generating binary image from ${UNIT_TEST_NAME}" + ) + add_custom_target(gen_binary_${UNIT_TEST_NAME} ALL DEPENDS "${build_dir}/${UNIT_TEST_NAME}.bin_timestamp") + + + ###################### Image executable in QEMU ################# + + + # A runnable image is a 4MB file with: + # bootloader at 0x1000 + # partition table at 0x8000 + # image at 0x10000 + add_custom_command( + OUTPUT "${CMAKE_CURRENT_BINARY_DIR}/${UNIT_TEST_NAME}.img_timestamp" + + COMMAND dd if=/dev/zero bs=1024 count=4096 of=${UNIT_TEST_NAME}.img + COMMAND dd if=${CMAKE_CURRENT_BINARY_DIR}/bootloader/bootloader.bin bs=1 seek=4096 of=${UNIT_TEST_NAME}.img conv=notrunc + COMMAND dd if=${CMAKE_CURRENT_BINARY_DIR}/partition_table/partition-table.bin bs=1 seek=32768 of=${UNIT_TEST_NAME}.img conv=notrunc + COMMAND dd if=${CMAKE_CURRENT_BINARY_DIR}/${UNIT_TEST_NAME}.bin bs=1 seek=65536 of=${UNIT_TEST_NAME}.img conv=notrunc + + COMMAND ${CMAKE_COMMAND} -E echo "Generated ${UNIT_TEST_NAME}.img" + COMMAND ${CMAKE_COMMAND} -E md5sum "${CMAKE_CURRENT_BINARY_DIR}/${UNIT_TEST_NAME}.img" > "${CMAKE_CURRENT_BINARY_DIR}/${UNIT_TEST_NAME}.img_timestamp" + + DEPENDS gen_binary_${UNIT_TEST_NAME} + VERBATIM + WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} + COMMENT "Generating binary image from ${UNIT_TEST_NAME}" + ) + add_custom_target(gen_image_${UNIT_TEST_NAME} ALL DEPENDS "${build_dir}/${UNIT_TEST_NAME}.img_timestamp") + + LIST(APPEND ESP32_TEST_IMAGES ${UNIT_TEST_NAME}.img) + + + # IMAGE CAN BE RUN AS + # $QEMU_ESP32 -nographic -no-reboot -machine esp32 -drive file=out/esp32-qemu-tests/testASN1.img,if=mtd,format=raw +endmacro() diff --git a/src/test_driver/esp32/dummy.c b/src/test_driver/esp32/dummy.c new file mode 100644 index 00000000000000..e69de29bb2d1d6 diff --git a/src/test_driver/esp32/main/CMakeLists.txt b/src/test_driver/esp32/main/CMakeLists.txt new file mode 100644 index 00000000000000..ded51eb78bee8c --- /dev/null +++ b/src/test_driver/esp32/main/CMakeLists.txt @@ -0,0 +1,42 @@ +# +# Copyright (c) 2021 Project CHIP Authors +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# (Uses default behaviour of compiling all source files in directory, adding 'include' to include path.) +# The list of src and include dirs must be in sync with that in all-clusters-app/esp32/main/component.mk +set(PRIV_INCLUDE_DIRS_LIST + "${CMAKE_CURRENT_LIST_DIR}/include" +) + +set(SRC_DIRS_LIST + "${CMAKE_CURRENT_LIST_DIR}" +) + +set(EXTRA_COMPONENT_DIRS + "${CMAKE_CURRENT_LIST_DIR}/../third_party/connectedhomeip/config/esp32/components" + "${IDF_PATH}/examples/common_components" +) + +if (CONFIG_OPENTHREAD_ENABLED) + list(APPEND PRIV_REQUIRES_LIST openthread) +endif() + +idf_component_register(PRIV_INCLUDE_DIRS ${PRIV_INCLUDE_DIRS_LIST} + SRC_DIRS ${SRC_DIRS_LIST} + PRIV_REQUIRES ${PRIV_REQUIRES_LIST}) + +set_property(TARGET ${COMPONENT_LIB} PROPERTY CXX_STANDARD 17) +target_compile_options(${COMPONENT_LIB} PRIVATE "-DLWIP_IPV6_SCOPES=0" "-DCHIP_HAVE_CONFIG_H") diff --git a/src/test_driver/esp32/main/component.mk b/src/test_driver/esp32/main/component.mk deleted file mode 100644 index e00ee9dea96ffc..00000000000000 --- a/src/test_driver/esp32/main/component.mk +++ /dev/null @@ -1,22 +0,0 @@ -# -# Copyright (c) 2020 Project CHIP Authors -# All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# Description: -# Component makefile for the ESP32 demo application. -# -# (Uses default behaviour of compiling all source files in directory, adding 'include' to include path.) - -COMPONENT_DEPENDS := chip diff --git a/src/test_driver/esp32/qemu_setup.sh b/src/test_driver/esp32/qemu_setup.sh deleted file mode 100755 index fac58a636271d7..00000000000000 --- a/src/test_driver/esp32/qemu_setup.sh +++ /dev/null @@ -1,40 +0,0 @@ -#!/usr/bin/env bash -# -# -# Copyright (c) 2020 Project CHIP Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# -# Description: -# This is a utility script that compiles ESP32 QEMU and sets it up -# for testing. -# - -here=$(cd "$(dirname "$0")" && pwd) - -set -e - -die() { - echo "${me:?}: *** ERROR: " "${*}" - exit 1 -} - -# move to the example folder, I don't work anywhere else -cd "$here" || die 'ack!, where am I?!?' - -# shellcheck source=/dev/null -source idf.sh -rm -f ./build/sdkconfig -SDKCONFIG=./build/sdkconfig SDKCONFIG_DEFAULTS=sdkconfig_qemu.defaults idf make defconfig -SDKCONFIG=./build/sdkconfig idf make -j8 esp32_elf_builder diff --git a/src/test_driver/esp32/run_qemu_image.py b/src/test_driver/esp32/run_qemu_image.py new file mode 100755 index 00000000000000..d871083a0c1c77 --- /dev/null +++ b/src/test_driver/esp32/run_qemu_image.py @@ -0,0 +1,133 @@ +#!/usr/bin/env python3 +# Copyright (c) 2021 Project CHIP Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import coloredlogs +import click +import logging +import os +import sys +import subprocess + +# Supported log levels, mapping string values required for argument +# parsing into logging constants +__LOG_LEVELS__ = { + 'debug': logging.DEBUG, + 'info': logging.INFO, + 'warn': logging.WARN, + 'fatal': logging.FATAL, +} + + +@click.command() +@click.option( + '--log-level', + default='INFO', + type=click.Choice(__LOG_LEVELS__.keys(), case_sensitive=False), + help='Determines the verbosity of script output.') +@click.option( + '--no-log-timestamps', + default=False, + is_flag=True, + help='Skip timestaps in log output') +@click.option( + '--image', + default=[], + multiple=True, + help='What images to execute (will be executed one after another)' +) +@click.option( + '--file-image-list', + default=None, + help='Read the images from the given file (contains images one per line)' +) +@click.option( + '--qemu', + default=os.environ.get('QEMU_ESP32', 'qemu-system-xtensa'), + help='QEMU binary to run (generally path to qemu-system-xtensa)' +) +@click.option( + '--verbose', + default=False, + is_flag=True, + help='More verbose output') +def main(log_level, no_log_timestamps, image, file_image_list, qemu, verbose): + # Ensures somewhat pretty logging of what is going on + log_fmt = '%(asctime)s %(levelname)-7s %(message)s' + if no_log_timestamps: + log_fmt = '%(levelname)-7s %(message)s' + coloredlogs.install(level=__LOG_LEVELS__[log_level], fmt=log_fmt) + + image = list(image) + + if file_image_list: + logging.info("Reading image list from %s", file_image_list) + + basedir = os.path.dirname(file_image_list) + + with open(file_image_list, 'rt') as f: + for name in f.readlines(): + name = name.strip() + + image_path = name + if not os.path.isabs(name): + image_path = os.path.join(basedir, name) + + logging.info(" Found %s => %s", name, image_path) + image.append(image_path) + + # the list "image" contains all the images that need to run + for path in image: + logging.info("Executing image %s", path) + + status = subprocess.run([qemu, "-nographic", "-no-reboot", "-machine", "esp32", + "-drive", "file=%s,if=mtd,format=raw" % path], capture_output=True) + + # Encoding is NOT valid, but want to not try to decode potential + # invalid UTF-8 sequences. The strings we care about are ascii anyway + output = status.stdout.decode('ascii') + + try: + if status.returncode != 0: + raise Exception("Execution of %s failed with code %d" % + (path, status.returncode)) + + # Parse output of the unit test. Generally expect things like: + # Failed Tests: 0 / 5 + # Failed Asserts: 0 / 77 + # I (3034) CHIP-tests: CHIP test status: 0 + for line in output.split('\n'): + if line.startswith('Failed Tests:') and not line.startswith('Failed Tests: 0 /'): + raise Exception("Tests seem failed: %s" % line) + + if line.startswith('Failed Asserts:') and not line.startswith('Failed Asserts: 0 /'): + raise Exception("Asserts seem failed: %s" % line) + + if 'CHIP test status: ' in line and 'CHIP test status: 0' not in line: + raise Exception("CHIP test status is NOT 0: %s" % line) + + if verbose: + print("========== TEST OUTPUT BEGIN ============") + print(output) + print("========== TEST OUTPUT END ============") + + logging.info("Image %s PASSED", path) + except: + # make sure output is visible in stdout + print(output) + raise + + +if __name__ == '__main__': + main() diff --git a/src/test_driver/esp32/sdkconfig_qemu.defaults b/src/test_driver/esp32/sdkconfig_qemu.defaults index 21cf961c44b80a..31a991644d1377 100644 --- a/src/test_driver/esp32/sdkconfig_qemu.defaults +++ b/src/test_driver/esp32/sdkconfig_qemu.defaults @@ -31,8 +31,8 @@ CONFIG_MBEDTLS_HARDWARE_MPI=n CONFIG_MBEDTLS_HARDWARE_SHA=n CONFIG_CHIP_TASK_STACK_SIZE=32768 CONFIG_ESP_MAIN_TASK_STACK_SIZE=32768 -CONFIG_ESP32_PANIC_PRINT_REBOOT=y -CONFIG_ESP32_PANIC_PRINT_HALT=n +CONFIG_ESP_SYSTEM_PANIC_PRINT_REBOOT=y +CONFIG_ESP_SYSTEM_PANIC_PRINT_HALT=n # enable BT CONFIG_BT_ENABLED=n diff --git a/src/transport/tests/qemu_transport_tests.sh b/src/transport/tests/qemu_transport_tests.sh deleted file mode 120000 index 8fbe617c97a267..00000000000000 --- a/src/transport/tests/qemu_transport_tests.sh +++ /dev/null @@ -1 +0,0 @@ -../../../scripts/tools/qemu_run_test.sh \ No newline at end of file