tests: use test_interactive_test_util when possible

dist/pythonlibs/testrunner/spawn.py

@@ -13,6 +13,7 @@
 import subprocess
 import time
 from traceback import extract_tb
+from . import utils
 
 PEXPECT_PATH = os.path.dirname(pexpect.__file__)
 RIOTBASE = (os.environ.get('RIOTBASE') or
@@ -50,6 +51,10 @@ def setup_child(timeout=10, spawnclass=pexpect.spawnu, env=None, logfile=None):
     except subprocess.CalledProcessError:
         # make reset yields error on some boards even if successful
         pass
+
+    # Handle synchronization if requested by the build system
+    sync_child(child)
+
     return child
 
 
@@ -60,3 +65,22 @@ def teardown_child(child):
         print("Process already stopped")
 
     child.close()
+
+
+def modules_list():
+    modules = set(os.environ.get('USEMODULE', '').split(' '))
+    modules.discard('')
+    return modules
+
+
+def sync_child(child):
+    # Do a child synchronization if used by a module
+    modules = modules_list()
+    _test_utils_interactive_sync(child, modules)
+
+
+def _test_utils_interactive_sync(child, modules, retries=5, delay=1):
+    if 'test_utils_interactive_sync' not in modules:
+        return
+
+    utils.test_utils_interactive_sync(child, retries=retries, delay=delay)

sys/auto_init/auto_init.c

@@ -96,6 +96,12 @@
 #include "schedstatistics.h"
 #endif
 
+#ifdef MODULE_TEST_UTILS_INTERACTIVE_SYNC
+#ifndef MODULE_SHELL_COMMANDS
+#include "test_utils/interactive_sync.h"
+#endif
+#endif
+
 #define ENABLE_DEBUG (0)
 #include "debug.h"
 
@@ -597,4 +603,10 @@ void auto_init(void)
     extern void suit_init_conditions(void);
     suit_init_conditions();
 #endif /* MODULE_SUIT */
+
+#ifdef MODULE_TEST_UTILS_INTERACTIVE_SYNC
+#ifndef MODULE_SHELL_COMMANDS
+    test_utils_interactive_sync();
+#endif
+#endif /* MODULE_TEST_UTILS_INTERACTIVE_SYNC */
 }

sys/shell/commands/Makefile

@@ -95,4 +95,8 @@ ifneq (,$(filter nimble_netif,$(USEMODULE)))
   SRC += sc_nimble_netif.c
 endif
 
+ifneq (,$(filter test_utils_interactive_sync,$(USEMODULE)))
+  SRC += sc_interactive_sync.c
+endif
+
 include $(RIOTBASE)/Makefile.base

sys/shell/commands/sc_interactive_sync.c

@@ -0,0 +1,39 @@
+/*
+ * Copyright (C) 2019 Inria
+ *
+ * This file is subject to the terms and conditions of the GNU Lesser
+ * General Public License v2.1. See the file LICENSE in the top level
+ * directory for more details.
+ */
+
+/**
+ * @ingroup     sys_shell_commands
+ * @{
+ *
+ * @file
+ * @brief       Shell commands interactive sync util
+ *
+ * @author      Francisco Molina <[email protected]>
+ * @}
+ */
+
+#include <stdio.h>
+#include "test_utils/interactive_sync.h"
+
+int _test_start(int argc, char **argv)
+{
+    (void) argc;
+    (void) argv;
+    puts("START");
+
+    return 0;
+}
+
+
+int _test_ready(int argc, char **argv)
+{
+    (void) argc;
+    (void) argv;
+    puts("READY");
+    return 0;
+}

sys/shell/commands/shell_commands.c

@@ -163,6 +163,11 @@ extern int _loramac_handler(int argc, char **argv);
 extern int _nimble_netif_handler(int argc, char **argv);
 #endif
 
+#ifdef MODULE_TEST_UTILS_INTERACTIVE_SYNC
+extern int _test_start(int argc, char **argv);
+extern int _test_ready(int argc, char **argv);
+#endif
+
 const shell_command_t _shell_command_list[] = {
     {"reboot", "Reboot the node", _reboot_handler},
 #ifdef MODULE_CONFIG
@@ -267,6 +272,10 @@ const shell_command_t _shell_command_list[] = {
 #endif
 #ifdef MODULE_NIMBLE_NETIF
     { "ble", "Manage BLE connections for NimBLE", _nimble_netif_handler },
+#endif
+#ifdef MODULE_TEST_UTILS_INTERACTIVE_SYNC
+    { "r", "Test sync, Ready query", _test_ready },
+    { "s", "Test sync, Start test trigger", _test_start },
 #endif
     {NULL, NULL, NULL}
 };

sys/test_utils/interactive_sync/interactive_sync.c

@@ -1,3 +1,22 @@
+/*
+ * Copyright (C) 2019 Freie Universität Berlin
+ *
+ * This file is subject to the terms and conditions of the GNU Lesser
+ * General Public License v2.1. See the file LICENSE in the top level
+ * directory for more details.
+ */
+
+/**
+ * @ingroup     sys
+ * @{
+ *
+ * @file
+ * @brief       Interactive Sync implementation
+ *
+ *
+ * @author      Gaëtan Harter <[email protected]>
+ */
+
 #include <stdio.h>
 #include "test_utils/interactive_sync.h"
 

tests/Makefile.tests_common

@@ -1,4 +1,7 @@
 APPLICATION ?= tests_$(notdir $(patsubst %/,%,$(CURDIR)))
+
+DEFAULT_MODULE += test_utils_interactive_sync
+
 ifneq (,$(filter tests_driver_%,$(APPLICATION)))
   BOARD ?= samr21-xpro
 endif

tests/README.md

@@ -7,6 +7,26 @@ allow basic functionality to be verified as well as provide an example of
 usage.
 
 
+Implementing automated tests
+----------------------------
+
+The goal is to be able to run all tests in a sequential way for as many targets
+as possible.
+
+As some board can't be reset without a manual trigger tests should be implemented
+with some kind of `synchronization`. This can be done in two ways:
+
+- use `test_utils_interactive_sync` when uart input/output does not need to be
+  disabled for the test. This is enabled by default.
+- set up the test in a loop so the test script will be able so sync with some kind
+  of start condition in the test.
+
+The module for the first option is `test_utils_interactive_sync` and is set as a
+default module in `Makefile.tests_common`. It can be disabled by setting in the
+application makefile `DISABLE_MODULE += test_utils_interactive_sync`. The python
+test script will adapt to it automatically.
+
+
 Running automated tests
 -----------------------
 

tests/bench_msg_pingpong/Makefile.ci

@@ -1,4 +1,8 @@
 BOARD_INSUFFICIENT_MEMORY := \
+    arduino-duemilanove \
+    arduino-nano \
+    arduino-uno \
+    atmega328p \
     nucleo-f031k6 \
     stm32f030f4-demo \
     #

tests/bench_mutex_pingpong/Makefile.ci

@@ -1,4 +1,8 @@
 BOARD_INSUFFICIENT_MEMORY := \
+    arduino-duemilanove \
+    arduino-nano \
+    arduino-uno \
+    atmega328p \
     nucleo-f031k6 \
     stm32f030f4-demo \
     #

tests/bench_thread_flags_pingpong/Makefile.ci

@@ -1,4 +1,8 @@
 BOARD_INSUFFICIENT_MEMORY := \
+    arduino-duemilanove \
+    arduino-nano \
+    arduino-uno \
+    atmega328p \
     nucleo-f031k6 \
     stm32f030f4-demo \
     #

tests/bench_thread_yield_pingpong/Makefile.ci

@@ -1,4 +1,8 @@
 BOARD_INSUFFICIENT_MEMORY := \
+    arduino-duemilanove \
+    arduino-nano \
+    arduino-uno \
+    atmega328p \
     nucleo-f031k6 \
     stm32f030f4-demo \
     #

tests/bloom_bytes/main.c

@@ -22,6 +22,8 @@
 #include <string.h>
 #include <inttypes.h>
 
+#include "test_utils/interactive_sync.h"
+
 #include "xtimer.h"
 #include "fmt.h"
 
@@ -63,6 +65,8 @@ int main(void)
 
     bloom_init(&bloom, BLOOM_BITS, bf, hashes, BLOOM_HASHF);
 
+    test_utils_interactive_sync();
+
     printf("Testing Bloom filter.\n\n");
     printf("m: %" PRIu32 " k: %" PRIu32 "\n\n", (uint32_t) bloom.m,
            (uint32_t) bloom.k);

tests/cond_order/main.c

@@ -24,7 +24,6 @@
 #include "cond.h"
 #include "mutex.h"
 #include "thread.h"
-#include "test_utils/interactive_sync.h"
 
 #define THREAD_NUMOF            (5U)
 #define THREAD_FIRSTGROUP_NUMOF (3U)
@@ -63,8 +62,6 @@ int main(void)
     puts("Condition variable order test");
     puts("Please refer to the README.md for more information\n");
 
-    test_utils_interactive_sync();
-
     mutex_init(&testlock);
     cond_init(&testcond);
 

tests/cond_order/tests/01-run.py

@@ -10,8 +10,6 @@
 import os
 import sys
 
-from testrunner import test_utils_interactive_sync
-
 thread_prio = {
         3:  6,
         4:  4,
@@ -23,7 +21,6 @@
 
 
 def testfunc(child):
-    test_utils_interactive_sync(child)
 
     for k in thread_prio.keys():
         child.expect_exact("T{} (prio {}): waiting on condition variable now"

tests/cpp_ctors/main.c

@@ -9,6 +9,7 @@
 #include "embUnit.h"
 #include "tests-cpp_ctors.h"
 #include "thread.h" /* For thread_getpid() */
+#include "test_utils/interactive_sync.h"
 
 long tests_cpp_ctors_global_value(void);
 long tests_cpp_ctors_static_value(void);
@@ -58,6 +59,8 @@ Test *tests_cpp_ctors_tests(void)
 
 int main(void)
 {
+    test_utils_interactive_sync();
+
     TESTS_START();
     TESTS_RUN(tests_cpp_ctors_tests());
     TESTS_END();

tests/driver_ds1307/Makefile.ci

@@ -0,0 +1,6 @@
+BOARD_INSUFFICIENT_MEMORY := \
+    arduino-duemilanove \
+    arduino-nano \
+    arduino-uno \
+    atmega328p \
+    #

tests/driver_nrfmin/tests/01-run.py

@@ -5,7 +5,6 @@
 
 
 def testfunc(child):
-    child.expect("All up, running the shell now")
     child.sendline("ifconfig")
     child.expect(r"Iface\s+(\d+)\s+HWaddr:")
 

tests/driver_pca9685/Makefile.ci

@@ -1,5 +1,6 @@
 BOARD_INSUFFICIENT_MEMORY := \
     arduino-duemilanove \
+    arduino-leonardo \
     arduino-nano \
     arduino-uno \
     atmega328p \

tests/float/main.c

@@ -23,6 +23,8 @@
 #include <stdio.h>
 #include <math.h>
 
+#include "test_utils/interactive_sync.h"
+
 #include "board.h"
 
 /* as default we run the test 100k times */
@@ -34,6 +36,8 @@
 
 int main(void)
 {
+    test_utils_interactive_sync();
+
     double x = 1234567.0 / 1024.0;
 
     puts("Testing floating point arithmetic...\n");

tests/gnrc_ipv6_ext/Makefile

@@ -35,6 +35,8 @@ USEMODULE += shell
 USEMODULE += shell_commands
 USEMODULE += ps
 
+DISABLE_MODULE += test_utils_interactive_sync
+
 # The test requires some setup and to be run as root
 # So it cannot currently be run
 TEST_ON_CI_BLACKLIST += all

tests/gnrc_ipv6_ext_frag/Makefile

@@ -44,6 +44,8 @@ USEMODULE += shell
 USEMODULE += shell_commands
 USEMODULE += ps
 
+DISABLE_MODULE += test_utils_interactive_sync
+
 # The test requires some setup and to be run as root
 # So it cannot currently be run
 TEST_ON_CI_BLACKLIST += all

tests/gnrc_ndp/Makefile.ci

@@ -21,4 +21,5 @@ BOARD_INSUFFICIENT_MEMORY := \
     waspmote-pro \
     wsn430-v1_3b \
     wsn430-v1_4 \
+    z1 \
     #

tests/gnrc_rpl_srh/Makefile

@@ -39,6 +39,8 @@ USEMODULE += shell
 USEMODULE += shell_commands
 USEMODULE += ps
 
+DISABLE_MODULE += test_utils_interactive_sync
+
 # The test requires some setup and to be run as root
 # So it cannot currently be run
 TEST_ON_CI_BLACKLIST += all

tests/gnrc_sixlowpan_frag/main.c

@@ -25,6 +25,8 @@
 #include "net/gnrc/sixlowpan/frag/rb.h"
 #include "xtimer.h"
 
+#include "test_utils/interactive_sync.h"
+
 #define TEST_NETIF_HDR_SRC      { 0xb3, 0x47, 0x60, 0x49, \
                                   0x78, 0xfe, 0x95, 0x48 }
 #define TEST_NETIF_HDR_DST      { 0xa4, 0xf2, 0xd2, 0xc9, \
@@ -627,6 +629,8 @@ static void run_unittests(void)
 
 int main(void)
 {
+    test_utils_interactive_sync();
+
     /* no auto-init, so xtimer needs to be initialized manually*/
     xtimer_init();
     /* netreg requires queue, but queue size one should be enough for us */